public void Run() { this.output.WriteLine("TPL start"); this.CheckFileExists(); var f = new FileInfo(this.sampleLogFileName); this.fileSizeInBytes = f.Length; this.lineSizeInBytesSoFar = 0; this.lineCount = 0; var sw = new Stopwatch(); sw.Start(); var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 8 }; var ab = new ActionBlock<string>(s => this.ProcessLine(s), options); using (TextReader reader = File.OpenText(this.sampleLogFileName)) { string line; while ((line = reader.ReadLine()) != null) { ab.Post(line); } } ab.Complete(); ab.Completion.Wait(); sw.Stop(); this.ShowResults(); this.output.WriteLine(); this.output.WriteLine("TPL done in {0}", sw.Elapsed); }
/// <summary> /// Creates the Disk output action block. /// </summary> /// <param name="containerName">Name of the container.</param> /// <param name="extension">The extension.</param> /// <param name="context">The context.</param> /// <param name="options">The options.</param> /// <returns>ActionBlock<System.Byte[]>.</returns> public static ActionBlock<byte[]> Create(CloudStorageAccount account, string containerName, string extension, string partitionId, ExecutionDataflowBlockOptions options) { var container = GetOrCreateContainer(account, containerName); Logger.Information("Creating appender for {Partition} using blob container {Container}", partitionId, container.Uri.AbsoluteUri); return new ActionBlock<byte[]>(async data => { if (data.Length > 4194304) { Logger.Fatal("Maximum size for blob append is 4MB, buffer size is {BufferSize}", data.Length); return; } try { var blob = GetAppendBlob(container, extension, partitionId); Logger.Debug("Appending {Length} bytes to blob {Uri}", data.Length, blob.Uri.AbsoluteUri); using (Logger.BeginTimedOperation("AppendFromByteArrayAsync", blob.Uri.AbsoluteUri, LogEventLevel.Debug)) await blob.AppendFromByteArrayAsync(data, 0, data.Length); } catch (Exception ex) { Logger.Error(ex, "BlobAppender Append From Byte Array Partition " + partitionId); } }, options); }
public DataflowWebHookSenderTests() { _handlerMock = new HttpMessageHandlerMock(); _httpClient = new HttpClient(_handlerMock); _options = new ExecutionDataflowBlockOptions(); _loggerMock = new Mock<ILogger>(); }
public void Start() { var sink = new ActionBlock<PageResultMessage>((Action<PageResultMessage>)Sink); var source = new BufferBlock<GetPageMessage>(); var linkOptions = new DataflowLinkOptions {PropagateCompletion = false}; for (int i = 0; i < 10; i++) { var options = new ExecutionDataflowBlockOptions { BoundedCapacity = 1 }; var worker = new TransformBlock<GetPageMessage, PageResultMessage>( (Func<GetPageMessage, PageResultMessage>)Worker, options); source.LinkTo(worker, linkOptions); worker.LinkTo(sink, linkOptions); } foreach (var url in UrlList.Urls) { source.Post(new GetPageMessage{ Url = url }); } source.Complete(); sink.Completion.Wait(); }
public void ProduceLogs(int count, int buffSize) { var bufferOptions = new DataflowBlockOptions() { BoundedCapacity = buffSize }; var writerOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 10, MaxDegreeOfParallelism = 1, MaxMessagesPerTask = 10, SingleProducerConstrained = true }; LogGenerator g = new LogGenerator(); var file = new StreamWriter("basic.async.buff.log", false); BufferBlock<string> buffer = new BufferBlock<string>(bufferOptions); ActionBlock<string> writer = new ActionBlock<string>(s => file.WriteLine(s), writerOptions); buffer.LinkTo(writer, new DataflowLinkOptions() { PropagateCompletion = true }); for (int i = 0; i < count; i++) { g.Next(); var line = string.Format(g.FormatStr, g.Param1, g.Param2, g.Param3, g.Param4, g.Param5, g.Param6); writer.SendAsync(line).Wait(); } buffer.Complete(); Completed = writer.Completion.ContinueWith(t => file.Close()); }
/// <summary> /// Initialize a new instance of the <see cref="DataflowWebHookSender"/> with the given retry policy, <paramref name="options"/>, /// and <paramref name="httpClient"/>. This constructor is intended for unit testing purposes. /// </summary> internal DataflowWebHookSender( ILogger logger, IEnumerable<TimeSpan> retryDelays, ExecutionDataflowBlockOptions options, HttpClient httpClient, Action<WebHookWorkItem> onWebHookSuccess, Action<WebHookWorkItem> onWebHookFailure) : base(logger) { retryDelays = retryDelays ?? DefaultRetries; options = options ?? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DefaultMaxConcurrencyLevel }; _httpClient = httpClient ?? new HttpClient(); // Create the launch processors with the given retry delays _launchers = new ActionBlock<WebHookWorkItem>[1 + retryDelays.Count()]; int offset = 0; _launchers[offset++] = new ActionBlock<WebHookWorkItem>(async item => await LaunchWebHook(item), options); foreach (TimeSpan delay in retryDelays) { _launchers[offset++] = new ActionBlock<WebHookWorkItem>(async item => await DelayedLaunchWebHook(item, delay)); } string msg = string.Format(CultureInfo.CurrentCulture, CustomResources.Manager_Started, typeof(DataflowWebHookSender).Name, _launchers.Length); Logger.Info(msg); // Set handlers for testing purposes _onWebHookSuccess = onWebHookSuccess; _onWebHookFailure = onWebHookFailure; }
public static async Task <ActionBlock <T> > ParallelEnumerateAsync <T>(IEnumerable <T> items, CancellationToken cancellationToken, int maxDegreeOfParallelism, Func <T, CancellationToken, Task> func) { var dataflowBlockOptions = new System.Threading.Tasks.Dataflow.ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, CancellationToken = cancellationToken }; var workerBlock = new ActionBlock <T>(item => func(item, cancellationToken), dataflowBlockOptions); foreach (var item in items) { await Policy .Handle <Exception>() .WaitAndRetryAsync(new[] { TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(60) }) .ExecuteAsync(async(ct) => { var sent = await workerBlock.SendAsync(item, cancellationToken); if (!sent) { throw new Exception($"Could not process item: {item}"); } }, cancellationToken ); } workerBlock.Complete(); await workerBlock.Completion; return(workerBlock); }
public static TransformBlock<EventData, byte[]> Create(ExecutionDataflowBlockOptions options) { return new TransformBlock<EventData, byte[]>(data => { return data.GetBytes(); }, options); }
public void MagnesiumInit() { _dcpManager = new DcpManager(); _requestQueueOptions = new ExecutionDataflowBlockOptions(); _storeQueue = new ActionBlock<DataContainer>(dc => _dcpManager.StoreHandler(dc)); _parseQueue = new ActionBlock<DataResponse>(dresp => _dcpManager.ParseHandler(dresp, _storeQueue)); _requestQueue = new ActionBlock<DataRequest>(dr => _dcpManager.RequestHandler(dr, _parseQueue), _requestQueueOptions); //布隆过滤器, 大小10M, 容错率0.001 _bloomFilter = new BloomFilter<string>(10000000, 0.001f); }
/// <summary> /// Creates the event sender action block. /// </summary> /// <param name="hubClient">The hub client.</param> /// <param name="options">The options.</param> /// <returns>ActionBlock<System.Byte[]>.</returns> public static ActionBlock<byte[]> Create(EventHubClient hubClient, ExecutionDataflowBlockOptions options) { return new ActionBlock<byte[]>(data => { //var index = Array.IndexOf(data, (byte)'>'); //if (index < 0) return Task.FromResult(0); //var key = Encoding.UTF8.GetString(data, 0, index); var eventData = new EventData(data); return hubClient.SendAsync(eventData); }, options); }
public WebHookManagerTests() { _handlerMock = new HttpMessageHandlerMock(); _httpClient = new HttpClient(_handlerMock); _storeMock = new Mock<IWebHookStore>(); _storeMock.Setup<Task<ICollection<WebHook>>>(s => s.QueryWebHooksAsync(TestUser, new[] { "a1" })) .ReturnsAsync(new Collection<WebHook> { CreateWebHook() }) .Verifiable(); _options = new ExecutionDataflowBlockOptions(); _loggerMock = new Mock<ILogger>(); _response = new HttpResponseMessage(); }
/// <summary> /// Initializes a new instance of the <see cref="ThreadedLogger" /> class. /// </summary> /// <param name="log">The log.</param> public ThreadedLogger(Action<string> log) { Guard.NotNull(log); _CancellationTokenSource = new CancellationTokenSource(); var options = new ExecutionDataflowBlockOptions { CancellationToken = _CancellationTokenSource.Token, MaxDegreeOfParallelism = 1 }; _ActionBlock = new ActionBlock<string>(log, options); }
public AggregationResult(EntityMetadata metadata, AggregationConfig config, int keyIndex) { _metadata = metadata; _config = config; _keyIndex = keyIndex; _aggregators = config.Aggregators.Select(c => c.CreateAggregator()).ToArray(); _groupResults = keyIndex < metadata.KeyCount ? new ConcurrentDictionary<int, AggregationResult>() : null; _cts = new CancellationTokenSource(); var executionOptions = new ExecutionDataflowBlockOptions { CancellationToken = _cts.Token, SingleProducerConstrained = true }; _changeBlock = new ActionBlock<AggregationChange>((Action<AggregationChange>)ProcessChange, executionOptions); }
static IEnumerable<int[]> GetTaskIdsForExecutionsOptions ( ExecutionDataflowBlockOptions options) { var blockFactories = new Func<ConcurrentQueue<Tuple<int, int>>, ITargetBlock<int>>[] { q => new ActionBlock<int> ( i => q.Enqueue (Tuple.Create (i, Task.CurrentId.Value)), options), q => new TransformBlock<int, int> (i => { q.Enqueue (Tuple.Create (i, Task.CurrentId.Value)); return i; }, options), q => new TransformManyBlock<int, int> (i => { q.Enqueue (Tuple.Create (i, Task.CurrentId.Value)); return new[] { i }; }, options) }; foreach (var factory in blockFactories) { var queue = new ConcurrentQueue<Tuple<int, int>> (); var block = factory (queue); Assert.IsEmpty (queue); for (int i = 0; i < 100; i++) block.Post (i); block.Complete (); var source = block as ISourceBlock<int>; if (source != null) { Assert.IsFalse (block.Completion.Wait (100)); source.LinkTo (new BufferBlock<int> ()); } Assert.IsTrue (block.Completion.Wait (1000)); CollectionAssert.AreEquivalent ( Enumerable.Range (0, 100), queue.Select (t => t.Item1)); yield return queue.Select (t => t.Item2).ToArray (); } }
/// <summary> /// Creates the batch. /// </summary> /// <param name="hubClient">The hub client.</param> /// <param name="options">The options.</param> /// <returns>ActionBlock<IEnumerable<System.Byte[]>>.</returns> public static ActionBlock<IEnumerable<byte[]>> CreateBatch(EventHubClient hubClient, ExecutionDataflowBlockOptions options) { return new ActionBlock<IEnumerable<byte[]>>(async batch => { var size = 0; var output = batch.Select(data => { //var index = Array.IndexOf(data, (byte)'>'); //var key = index < 0 ? null : Encoding.UTF8.GetString(data, 0, index); size += data.Length; return new EventData(data);// { PartitionKey = key }; }).ToList(); Logger.Debug("Sending {Count} events ({Size} bytes)", output.Count, size); using (Logger.BeginTimedOperation("SendBatch (" + output.Count + ")", null, LogEventLevel.Debug)) await Task.WhenAll(output.GroupBy(x => x.PartitionKey).Select(x => hubClient.SendPartitionedBatchAsync(x, true))); }, options); }
public static void Example1() { var conf = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 4 }; ActionBlock<int> a = new ActionBlock<int>(i => { Thread.Sleep(500); Console.WriteLine(i); }, conf); TransformBlock<int, int> t = new TransformBlock<int, int>(i => i * 3); t.LinkTo(a); for (int i = 0; i < 12; i++) { t.Post(i); } }
public void Run() { var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 8 }; var tb = new TransformBlock<int, int>(i => i * 2); var ab = new ActionBlock<int>(i => this.Compute(i), options); tb.LinkTo(ab); for (var i = 0; i < 10; i++) { tb.Post(i); } tb.Complete(); tb.Completion.Wait(); Thread.Sleep(500); }
static IEnumerable<ITargetBlock<int>> GetExecutionBlocksWithAsyncAction ( Func<int, Task> action, ExecutionDataflowBlockOptions options) { yield return new ActionBlock<int> (action, options); yield return new TransformBlock<int, int> ( i => action (i).ContinueWith ( t => { t.Wait (); return i; }), options); yield return new TransformManyBlock<int, int> ( i => action (i).ContinueWith ( t => { t.Wait (); return Enumerable.Empty<int> (); }), options); }
/// <summary> /// Creates the Disk output action block. Returns length of data written. /// </summary> /// <param name="filePath">The file path.</param> /// <param name="extension">The extension.</param> /// <param name="partitionId">The partition identifier.</param> /// <param name="options">The options.</param> /// <returns>ActionBlock<System.Byte[]>.</returns> public static TransformBlock<byte[], long> Create(string filePath, string extension, string partitionId, ExecutionDataflowBlockOptions options) { return new TransformBlock<byte[], long>(async data => { var fileName = Path.Combine(filePath, string.Concat(DateTime.UtcNow.ToString("yyyyMMddHH"), 'p', partitionId, '.', extension)); Logger.Debug("Appending {Length} bytes to file {Uri}", data.Length, fileName); using (Logger.BeginTimedOperation("WriteAsync", fileName, LogEventLevel.Debug)) using (var outputStream = File.OpenWrite(fileName)) { outputStream.Position = outputStream.Length; await outputStream.WriteAsync(data, 0, data.Length); await outputStream.FlushAsync(); } return data.Length; }, options); }
public async Task StartAsync(CancellationToken cancellationToken) { ThrowIfDisposed(); if (_watcher != null && _watcher.EnableRaisingEvents) { throw new InvalidOperationException("The listener has already been started."); } if (string.IsNullOrEmpty(_config.RootPath) || !Directory.Exists(_config.RootPath)) { throw new InvalidOperationException(string.Format("Path '{0}' is invalid. FilesConfiguration.RootPath must be set to a valid directory location.", _config.RootPath)); } CreateFileWatcher(); FileProcessorFactoryContext context = new FileProcessorFactoryContext(_config, _attribute, _triggerExecutor, _trace); _processor = _config.ProcessorFactory.CreateFileProcessor(context); ExecutionDataflowBlockOptions options = new ExecutionDataflowBlockOptions { BoundedCapacity = _processor.MaxQueueSize, MaxDegreeOfParallelism = _processor.MaxDegreeOfParallelism, }; _workQueue = new ActionBlock<FileSystemEventArgs>(async (e) => await ProcessWorkItem(e), options); // on startup, process any preexisting files that haven't been processed yet ProcessFiles(); // Create a timer to cleanup processed files. // The timer doesn't auto-reset. It will reset itself // when we receive more file events _cleanupTimer = new System.Timers.Timer() { AutoReset = false, Interval = _rand.Next(5 * 1000, 8 * 1000) }; _cleanupTimer.Elapsed += OnCleanupTimer; _cleanupTimer.Start(); await Task.FromResult<bool>(true); }
/// <summary> /// 初期化処理。 /// </summary> /// <param name="settingObject"></param> /// <param name="token"></param> /// <returns></returns> public void Init(dynamic settingObject, CancellationToken token) { logger.Trace("Init Start"); token.Register(Dispose); var opt = new DataflowBlockOptions { CancellationToken = token, }; var buffer = new BufferBlock<PastaLog>(opt); var actOpt = new ExecutionDataflowBlockOptions() { CancellationToken = token, SingleProducerConstrained = true, }; var act = new ActionBlock<PastaLog>((a) => Save(a), actOpt); buffer.LinkTo(act); Target = buffer; logger.Trace("Init End"); }
public static void Example2() { // Scenario: one buffer and 3 actions connected to it - each number will be processed only by one action var conf = new ExecutionDataflowBlockOptions(); // todo: play with those conf.BoundedCapacity = 1; conf.MaxDegreeOfParallelism = 4; var buffer = new BufferBlock<int>(); var action1 = new ActionBlock<int>(a => { Thread.Sleep(50); Console.WriteLine("Action 1 value: {0}", a); }, conf); var action2 = new ActionBlock<int>(a => { Thread.Sleep(50); Console.WriteLine("Action 2 value: {0}", a); }, conf); var action3 = new ActionBlock<int>(a => { Thread.Sleep(50); Console.WriteLine("Action 3 value: {0}", a); }, conf); buffer.LinkTo(action1); buffer.LinkTo(action2); buffer.LinkTo(action3); var t = new Task(() => { for (int i = 0; i < 12; i++) { buffer.Post(i); } }); t.Start(); }
public static void Example3() { // Scenario: one buffer and 3 actions connected to it - each number precessed is send to each action var conf = new ExecutionDataflowBlockOptions(); //conf.MaxDegreeOfParallelism = 4; var buffer = new BroadcastBlock<int>(i => i); var action1 = new ActionBlock<int>(a => { Thread.Sleep(50); Console.WriteLine("Action 1 value: {0}", a); }, conf); var action2 = new ActionBlock<int>(a => { Thread.Sleep(50); Console.WriteLine("Action 2 value: {0}", a); }, conf); var action3 = new ActionBlock<int>(a => { Thread.Sleep(50); Console.WriteLine("Action 3 value: {0}", a); }, conf); buffer.LinkTo(action1); buffer.LinkTo(action2); buffer.LinkTo(action3); var t = new Task(() => { for (int i = 0; i < 12; i++) { buffer.Post(i); } }); t.Start(); }
/// <summary> /// Instantiate a message processor, using the provided handlers /// and timeout/concurrency configuration /// </summary> public EventProcessor(IMessageHandlerResolver handlerResolver, ICircuitBreaker circuitBreaker, int maxConcurrency, string eventHubName, IDispatcherInstrumentationPublisher instrumentationPublisher) { Logger.Info( "Initializing event processor with concurrency {0}", maxConcurrency); _handlerResolver = handlerResolver; _circuitBreaker = circuitBreaker; _eventHubName = eventHubName; _instrumentationPublisher = instrumentationPublisher; // Set up the execution options for bounded concurrency // when using the TPL data flow action block _options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxConcurrency }; }
public void Run() { var propagateCompletion = new DataflowLinkOptions { PropagateCompletion = true }; var csep = new ConcurrentExclusiveSchedulerPair(); var concurrentOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, TaskScheduler = csep.ConcurrentScheduler }; var exclusiveOptions = new ExecutionDataflowBlockOptions { TaskScheduler = csep.ExclusiveScheduler }; var concurrent = new ActionBlock<int>(async i => { Console.WriteLine("Concurent print value: {0}", i); await Task.Delay(500); }, concurrentOptions); var exclusive = new ActionBlock<int>(i => { Console.WriteLine("Exclusive print value: {0}", i); Thread.Sleep(750); }, exclusiveOptions); var broadcaster = new BroadcastBlock<int>(i => i); broadcaster.LinkTo(concurrent, propagateCompletion); broadcaster.LinkTo(exclusive, propagateCompletion, i => i % 2 == 0); Enumerable .Range(1, 10) .ToList() .ForEach(i => broadcaster.Post(i)); broadcaster.Complete(); Task.WaitAll(concurrent.Completion, exclusive.Completion); }
/// <summary> /// Creates the Transformation Block to BZip2 compress. /// </summary> /// <param name="options">The options.</param> public static TransformBlock<byte[], byte[]> Create(ExecutionDataflowBlockOptions options) { return new TransformBlock<byte[], byte[]>(data => { using (var output = new MemoryStream()) { using (Logger.BeginTimedOperation("BZip2OutputStream", null, LogEventLevel.Debug)) using (var input = new MemoryStream(data)) using (var compression = new BZip2OutputStream(output)) { input.CopyTo(compression); compression.Flush(); } var result = output.ToArray(); Logger.Debug("Compressed {InputSize} bytes to {OutputSize} ({Compression:00}%)", data.Length, result.Length, result.Length * 100M/data.Length); return result; } }, options); }
public void MaxDegreeOfParallelismTest() { // loop to better test for race conditions // some that showed in this test were quite rare for (int i = 0; i < 10; i++) { var options = new ExecutionDataflowBlockOptions (); foreach (var taskIds in GetTaskIdsForExecutionsOptions(options)) Assert.AreEqual (1, CalculateDegreeOfParallelism (taskIds)); options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 }; foreach (var taskIds in GetTaskIdsForExecutionsOptions (options)) Assert.LessOrEqual (CalculateDegreeOfParallelism (taskIds), 2); options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 4 }; foreach (var taskIds in GetTaskIdsForExecutionsOptions (options)) Assert.LessOrEqual (CalculateDegreeOfParallelism (taskIds), 4); options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = -1 }; foreach (var taskIds in GetTaskIdsForExecutionsOptions (options)) Assert.LessOrEqual (CalculateDegreeOfParallelism (taskIds), taskIds.Length); } }
/// <summary> /// Initializes the <see cref="TransformBlock{TInput,TOutput}"/> with the specified <see cref="System.Func{TInput,TOutput}"/> /// and <see cref="DataflowBlockOptions"/>. /// </summary> /// <param name="transformSync">The synchronous function to invoke with each data element received.</param> /// <param name="transformAsync">The asynchronous function to invoke with each data element received.</param> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="TransformBlock{TInput,TOutput}"/>.</param> /// <exception cref="System.ArgumentNullException">The <paramref name="transformSync"/> and <paramref name="transformAsync"/> are both null (Nothing in Visual Basic).</exception> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> private TransformBlock(Func <TInput, TOutput> transformSync, Func <TInput, Task <TOutput> > transformAsync, ExecutionDataflowBlockOptions dataflowBlockOptions) { if (transformSync == null && transformAsync == null) { throw new ArgumentNullException("transform"); } if (dataflowBlockOptions == null) { throw new ArgumentNullException("dataflowBlockOptions"); } Contract.Requires(transformSync == null ^ transformAsync == null, "Exactly one of transformSync and transformAsync must be null."); Contract.EndContractBlock(); // Ensure we have options that can't be changed by the caller dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone(); // Initialize onItemsRemoved delegate if necessary Action <ISourceBlock <TOutput>, int> onItemsRemoved = null; if (dataflowBlockOptions.BoundedCapacity > 0) { onItemsRemoved = (owningSource, count) => ((TransformBlock <TInput, TOutput>)owningSource)._target.ChangeBoundingCount(-count); } // Initialize source component. _source = new SourceCore <TOutput>(this, dataflowBlockOptions, owningSource => ((TransformBlock <TInput, TOutput>)owningSource)._target.Complete(exception: null, dropPendingMessages: true), onItemsRemoved); // If parallelism is employed, we will need to support reordering messages that complete out-of-order if (dataflowBlockOptions.SupportsParallelExecution) { _reorderingBuffer = new ReorderingBuffer <TOutput>(this, (owningSource, message) => ((TransformBlock <TInput, TOutput>)owningSource)._source.AddMessage(message)); } // Create the underlying target if (transformSync != null) // sync { _target = new TargetCore <TInput>(this, messageWithId => ProcessMessage(transformSync, messageWithId), _reorderingBuffer, dataflowBlockOptions, TargetCoreOptions.None); } else // async { Contract.Assert(transformAsync != null, "Incorrect delegate type."); _target = new TargetCore <TInput>(this, messageWithId => ProcessMessageWithTask(transformAsync, messageWithId), _reorderingBuffer, dataflowBlockOptions, TargetCoreOptions.UsesAsyncCompletion); } // Link up the target half with the source half. In doing so, // ensure exceptions are propagated, and let the source know no more messages will arrive. // As the target has completed, and as the target synchronously pushes work // through the reordering buffer when async processing completes, // we know for certain that no more messages will need to be sent to the source. #if PRENET45 m_target.Completion.ContinueWith(completed => { if (completed.IsFaulted) { m_source.AddAndUnwrapAggregateException(completed.Exception); } m_source.Complete(); }, CancellationToken.None, Common.GetContinuationOptions(), TaskScheduler.Default); #else _target.Completion.ContinueWith((completed, state) => { var sourceCore = (SourceCore <TOutput>)state; if (completed.IsFaulted) { sourceCore.AddAndUnwrapAggregateException(completed.Exception); } sourceCore.Complete(); }, _source, CancellationToken.None, Common.GetContinuationOptions(), TaskScheduler.Default); #endif // It is possible that the source half may fault on its own, e.g. due to a task scheduler exception. // In those cases we need to fault the target half to drop its buffered messages and to release its // reservations. This should not create an infinite loop, because all our implementations are designed // to handle multiple completion requests and to carry over only one. #if PRENET45 m_source.Completion.ContinueWith(completed => { Contract.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion."); (this as IDataflowBlock).Fault(completed.Exception); }, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); #else _source.Completion.ContinueWith((completed, state) => { var thisBlock = ((TransformBlock <TInput, TOutput>)state) as IDataflowBlock; Contract.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion."); thisBlock.Fault(completed.Exception); }, this, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); #endif // Handle async cancellation requests by declining on the target Common.WireCancellationToComplete( dataflowBlockOptions.CancellationToken, Completion, state => ((TargetCore <TInput>)state).Complete(exception: null, dropPendingMessages: true), _target); #if FEATURE_TRACING var etwLog = DataflowEtwProvider.Log; if (etwLog.IsEnabled()) { etwLog.DataflowBlockCreated(this, dataflowBlockOptions); } #endif }
/// <summary>Initializes the <see cref="TransformManyBlock{TInput,TOutput}"/> with the specified function and <see cref="ExecutionDataflowBlockOptions"/>.</summary> /// <param name="transform"> /// The function to invoke with each data element received. All of the data asynchronously returned in the <see cref="System.Collections.Generic.IEnumerable{TOutput}"/> /// will be made available as output from this <see cref="TransformManyBlock{TInput,TOutput}"/>. /// </param> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="TransformManyBlock{TInput,TOutput}"/>.</param> /// <exception cref="System.ArgumentNullException">The <paramref name="transform"/> is null (Nothing in Visual Basic).</exception> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> public TransformManyBlock(Func <TInput, Task <IEnumerable <TOutput> > > transform, ExecutionDataflowBlockOptions dataflowBlockOptions) { if (transform == null) { throw new ArgumentNullException(nameof(transform)); } Initialize(messageWithId => ProcessMessageWithTask(transform, messageWithId), dataflowBlockOptions, ref _source, ref _target, ref _reorderingBuffer, TargetCoreOptions.UsesAsyncCompletion); }
//[Fact(Skip = "Outerloop")] public void RunTransformBlockConformanceTests() { bool passed = true; // SYNC #region Sync { // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++) { Func<DataflowBlockOptions, TargetProperties<int>> transformBlockFactory = options => { TransformBlock<int, int> transformBlock = new TransformBlock<int, int>(i => i, (ExecutionDataflowBlockOptions)options); ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); transformBlock.LinkTo(actionBlock); return new TargetProperties<int> { Target = transformBlock, Capturer = actionBlock, ErrorVerifyable = false }; }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2, CancellationToken = cancellationSource.Token }; passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformBlockFactory, dopOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformBlockFactory, mptOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformBlockFactory, mptOptions, 1, Intervention.Complete, null, feedMethod, true); passed &= FeedTarget(transformBlockFactory, cancellationOptions, 1, Intervention.Cancel, cancellationSource, feedMethod, true); } // Test chained Post/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2)); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2)); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained Post all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2)); for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true; for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16; Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2)); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i); Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive(); localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16)); Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that OperationCanceledExceptions are ignored { bool localPassed = true; var t = new TransformBlock<int, int>(i => { if ((i % 2) == 0) throw new OperationCanceledException(); return i; }); for (int i = 0; i < 2; i++) t.Post(i); t.Complete(); for (int i = 0; i < 2; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test using a precanceled token { bool localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token }; var t = new TransformBlock<int, int>(i => i, dbo); int ignoredValue; IList<int> ignoredValues; localPassed &= t.LinkTo(new ActionBlock<int>(delegate { })) != null; localPassed &= t.SendAsync(42).Result == false; localPassed &= t.TryReceiveAll(out ignoredValues) == false; localPassed &= t.Post(42) == false; localPassed &= t.OutputCount == 0; localPassed &= t.TryReceive(out ignoredValue) == false; localPassed &= t.Completion != null; t.Complete(); } catch (Exception) { localPassed = false; } Console.WriteLine(" {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting { bool localPassed = true; var t = new TransformBlock<int, int>(new Func<int, int>(i => { throw new InvalidOperationException(); })); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" {0}: Faulted handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } } #endregion #region Async // ASYNC (a copy of the sync but with constructors returning Task<T> instead of T { // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++) { Func<DataflowBlockOptions, TargetProperties<int>> transformBlockFactory = options => { TransformBlock<int, int> transformBlock = new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i), (ExecutionDataflowBlockOptions)options); ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); transformBlock.LinkTo(actionBlock); return new TargetProperties<int> { Target = transformBlock, Capturer = actionBlock, ErrorVerifyable = false }; }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2, CancellationToken = cancellationSource.Token }; passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformBlockFactory, dopOptions, 10, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true); passed &= FeedTarget(transformBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true); } // Test chained Post/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2))); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2))); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained Post all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2))); for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true; for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16; Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2))); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i); Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive(); localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16)); Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that OperationCanceledExceptions are ignored { bool localPassed = true; var t = new TransformBlock<int, int>(i => { if ((i % 2) == 0) throw new OperationCanceledException(); return Task.Factory.StartNew(() => i); }); for (int i = 0; i < 2; i++) t.Post(i); t.Complete(); for (int i = 0; i < 2; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that null tasks are ignored { bool localPassed = true; var t = new TransformBlock<int, int>(i => { if ((i % 2) == 0) return null; return Task.Factory.StartNew(() => i); }); for (int i = 0; i < 2; i++) t.Post(i); t.Complete(); for (int i = 0; i < 2; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: null tasks are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that null tasks are ignored when a reordering buffer is in place { bool localPassed = true; var t = new TransformBlock<int, int>(i => { if (i == 0) { Task.Delay(10).Wait(); return null; } return Task.Factory.StartNew(() => i); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 }); t.Post(0); t.Post(1); try { localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1; } catch { localPassed = false; } Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting from the delegate { bool localPassed = true; var t = new TransformBlock<int, int>(new Func<int, Task<int>>(i => { throw new InvalidOperationException(); })); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting from the task { bool localPassed = true; var t = new TransformBlock<int, int>(new Func<int, Task<int>>(i => Task<int>.Factory.StartNew(() => { throw new InvalidOperationException(); }))); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } } #endregion Assert.True(passed, "Test failed."); }
/// <summary>Initializes the <see cref="ActionBlock{T}"/> with the specified <see cref="System.Func{T,Task}"/> and <see cref="ExecutionDataflowBlockOptions"/>.</summary> /// <param name="action">The action to invoke with each data element received.</param> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="ActionBlock{T}"/>.</param> /// <exception cref="System.ArgumentNullException">The <paramref name="action"/> is null (Nothing in Visual Basic).</exception> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> public ActionBlock(Func <TInput, Task> action, ExecutionDataflowBlockOptions dataflowBlockOptions) : this((Delegate)action, dataflowBlockOptions) { }
/// <summary>Initializes the <see cref="ActionBlock{T}"/> with the specified delegate and options.</summary> /// <param name="action">The action to invoke with each data element received.</param> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="ActionBlock{T}"/>.</param> /// <exception cref="System.ArgumentNullException">The <paramref name="action"/> is null (Nothing in Visual Basic).</exception> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> private ActionBlock(Delegate action, ExecutionDataflowBlockOptions dataflowBlockOptions) { // Validate arguments if (action == null) { throw new ArgumentNullException(nameof(action)); } if (dataflowBlockOptions == null) { throw new ArgumentNullException(nameof(dataflowBlockOptions)); } // Ensure we have options that can't be changed by the caller dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone(); // Based on the mode, initialize the target. If the user specifies SingleProducerConstrained, // we'll try to employ an optimized mode under a limited set of circumstances. var syncAction = action as Action <TInput>; if (syncAction != null && dataflowBlockOptions.SingleProducerConstrained && dataflowBlockOptions.MaxDegreeOfParallelism == 1 && !dataflowBlockOptions.CancellationToken.CanBeCanceled && dataflowBlockOptions.BoundedCapacity == DataflowBlockOptions.Unbounded) { // Initialize the SPSC fast target to handle the bulk of the processing. // The SpscTargetCore is only supported when BoundedCapacity, CancellationToken, // and MaxDOP are all their default values. It's also only supported for sync // delegates and not for async delegates. _spscTarget = new SpscTargetCore <TInput>(this, syncAction, dataflowBlockOptions); } else { // Initialize the TargetCore which handles the bulk of the processing. // The default target core can handle all options and delegate flavors. if (syncAction != null) // sync { _defaultTarget = new TargetCore <TInput>(this, messageWithId => ProcessMessage(syncAction, messageWithId), null, dataflowBlockOptions, TargetCoreOptions.RepresentsBlockCompletion); } else // async { var asyncAction = action as Func <TInput, Task>; Debug.Assert(asyncAction != null, "action is of incorrect delegate type"); _defaultTarget = new TargetCore <TInput>(this, messageWithId => ProcessMessageWithTask(asyncAction, messageWithId), null, dataflowBlockOptions, TargetCoreOptions.RepresentsBlockCompletion | TargetCoreOptions.UsesAsyncCompletion); } // Handle async cancellation requests by declining on the target Common.WireCancellationToComplete( dataflowBlockOptions.CancellationToken, Completion, state => ((TargetCore <TInput>)state).Complete(exception: null, dropPendingMessages: true), _defaultTarget); } #if FEATURE_TRACING DataflowEtwProvider etwLog = DataflowEtwProvider.Log; if (etwLog.IsEnabled()) { etwLog.DataflowBlockCreated(this, dataflowBlockOptions); } #endif Debug.Assert((_spscTarget != null) ^ (_defaultTarget != null), "One and only one of the two targets must be non-null after construction"); }
/// <summary> /// Initializes the <see cref="TransformBlock{TInput,TOutput}"/> with the specified <see cref="System.Func{TInput,TOutput}"/> /// and <see cref="ExecutionDataflowBlockOptions"/>. /// </summary> /// <param name="transform">The function to invoke with each data element received.</param> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="TransformBlock{TInput,TOutput}"/>.</param> /// <exception cref="System.ArgumentNullException">The <paramref name="transform"/> is null (Nothing in Visual Basic).</exception> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> public TransformBlock(Func <TInput, Task <TOutput> > transform, ExecutionDataflowBlockOptions dataflowBlockOptions) : this(null, transform, dataflowBlockOptions) { }
/// <summary>Initializes the <see cref="TransformManyBlock{TInput,TOutput}"/> with the specified function and <see cref="ExecutionDataflowBlockOptions"/>.</summary> /// <param name="transformSync">The synchronous function to invoke with each data element received.</param> /// <param name="transformAsync">The asynchronous function to invoke with each data element received.</param> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="TransformManyBlock{TInput,TOutput}"/>.</param> /// <exception cref="System.ArgumentNullException">The <paramref name="transformSync"/> and <paramref name="transformAsync"/> are both null (Nothing in Visual Basic).</exception> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> private TransformManyBlock(Func <TInput, IEnumerable <TOutput> >?transformSync, Func <TInput, Task <IEnumerable <TOutput> > >?transformAsync, ExecutionDataflowBlockOptions dataflowBlockOptions) { // Validate arguments. It's ok for the filterFunction to be null, but not the other parameters. if (transformSync == null && transformAsync == null) { throw new ArgumentNullException("transform"); } if (dataflowBlockOptions == null) { throw new ArgumentNullException(nameof(dataflowBlockOptions)); } Debug.Assert(transformSync == null ^ transformAsync == null, "Exactly one of transformSync and transformAsync must be null."); // Ensure we have options that can't be changed by the caller dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone(); // Initialize onItemsRemoved delegate if necessary Action <ISourceBlock <TOutput>, int>?onItemsRemoved = null; if (dataflowBlockOptions.BoundedCapacity > 0) { onItemsRemoved = (owningSource, count) => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.ChangeBoundingCount(-count); } // Initialize source component _source = new SourceCore <TOutput>(this, dataflowBlockOptions, owningSource => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.Complete(exception: null, dropPendingMessages: true), onItemsRemoved); // If parallelism is employed, we will need to support reordering messages that complete out-of-order. // However, a developer can override this with EnsureOrdered == false. if (dataflowBlockOptions.SupportsParallelExecution && dataflowBlockOptions.EnsureOrdered) { _reorderingBuffer = new ReorderingBuffer <IEnumerable <TOutput> >( this, (source, messages) => ((TransformManyBlock <TInput, TOutput>)source)._source.AddMessages(messages)); } // Create the underlying target and source if (transformSync != null) // sync { // If an enumerable function was provided, we can use synchronous completion, meaning // that the target will consider a message fully processed as soon as the // delegate returns. _target = new TargetCore <TInput>(this, messageWithId => ProcessMessage(transformSync, messageWithId), _reorderingBuffer, dataflowBlockOptions, TargetCoreOptions.None); } else // async { Debug.Assert(transformAsync != null, "Incorrect delegate type."); // If a task-based function was provided, we need to use asynchronous completion, meaning // that the target won't consider a message completed until the task // returned from that delegate has completed. _target = new TargetCore <TInput>(this, messageWithId => ProcessMessageWithTask(transformAsync, messageWithId), _reorderingBuffer, dataflowBlockOptions, TargetCoreOptions.UsesAsyncCompletion); } // Link up the target half with the source half. In doing so, // ensure exceptions are propagated, and let the source know no more messages will arrive. // As the target has completed, and as the target synchronously pushes work // through the reordering buffer when async processing completes, // we know for certain that no more messages will need to be sent to the source. _target.Completion.ContinueWith((completed, state) => { var sourceCore = (SourceCore <TOutput>)state !; if (completed.IsFaulted) { sourceCore.AddAndUnwrapAggregateException(completed.Exception !); } sourceCore.Complete(); }, _source, CancellationToken.None, Common.GetContinuationOptions(), TaskScheduler.Default); // It is possible that the source half may fault on its own, e.g. due to a task scheduler exception. // In those cases we need to fault the target half to drop its buffered messages and to release its // reservations. This should not create an infinite loop, because all our implementations are designed // to handle multiple completion requests and to carry over only one. _source.Completion.ContinueWith((completed, state) => { var thisBlock = ((TransformManyBlock <TInput, TOutput>)state !) as IDataflowBlock; Debug.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion."); thisBlock.Fault(completed.Exception !); }, this, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); // Handle async cancellation requests by declining on the target Common.WireCancellationToComplete( dataflowBlockOptions.CancellationToken, Completion, state => ((TargetCore <TInput>)state !).Complete(exception: null, dropPendingMessages: true), _target); #if FEATURE_TRACING DataflowEtwProvider etwLog = DataflowEtwProvider.Log; if (etwLog.IsEnabled()) { etwLog.DataflowBlockCreated(this, dataflowBlockOptions); } #endif }
public void MaxMessagesPerTaskTest() { var options = new ExecutionDataflowBlockOptions (); foreach (var taskIds in GetTaskIdsForExecutionsOptions (options)) Assert.GreaterOrEqual (taskIds.Distinct ().Count (), 1); options = new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1 }; foreach (var taskIds in GetTaskIdsForExecutionsOptions (options)) Assert.AreEqual (100, taskIds.Distinct ().Count ()); options = new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 2 }; foreach (var taskIds in GetTaskIdsForExecutionsOptions (options)) Assert.GreaterOrEqual (taskIds.Distinct ().Count (), taskIds.Length / 2); options = new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 4 }; foreach (var taskIds in GetTaskIdsForExecutionsOptions (options)) Assert.GreaterOrEqual (taskIds.Distinct ().Count (), taskIds.Length / 4); }
/// <summary> /// Create an adapter to change T into a dataflow message /// </summary> /// <param name="options">Execution options for the returned block</param> /// <returns>The adapter block</returns> public static IPropagatorBlock <T, DataflowMessage <T> > CreateAdapter(ExecutionDataflowBlockOptions options) => new TransformBlock <T, DataflowMessage <T> >(arg => _pool.Get().Init(arg), options);
/// <summary>Initializes the <see cref="ActionBlock{T}"/> with the specified <see cref="System.Action{T}"/> and <see cref="ExecutionDataflowBlockOptions"/>.</summary> /// <param name="action">The action to invoke with each data element received.</param> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="ActionBlock{T}"/>.</param> /// <exception cref="System.ArgumentNullException">The <paramref name="action"/> is null (Nothing in Visual Basic).</exception> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> public ActionBlock(Action <TInput> action, ExecutionDataflowBlockOptions dataflowBlockOptions) : this((Delegate)action, dataflowBlockOptions) { }
private static bool TestQuickStop(bool testThrow) { bool passed = true; CancellationTokenSource cts = new CancellationTokenSource(); var options = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token }; var propagator = new TransformBlock<ThrowOn, ThrowOn>(x => { Task.Delay(200).Wait(); return x; }, options); var thrower = new ThrowerBlock(); // Post enough messages to require long processing for (int i = 0; i < 2; i++) propagator.Post(testThrow && i == 1 ? ThrowOn.OfferMessage : ThrowOn.TryReceive); // Throw on the second message // Link the thrower propagator.LinkTo(thrower); // Once a message has been processed, cancel the propagator (if we are testing cancellation) SpinWait.SpinUntil(() => thrower.LastOperation == ThrowOn.OfferMessage); if (!testThrow) cts.Cancel(); // Wait for the propagator to complete try { var ranToCompletion = propagator.Completion.Wait(10000); passed = false; Console.WriteLine("Task is faulted or canceled (finished: {0}) - FAILED", ranToCompletion ? "ran to copmpletion" : "still running"); } catch (AggregateException ae) { passed = testThrow ? ae.InnerException is InvalidOperationException : ae.InnerException is TaskCanceledException; ae.Handle(e => true); Console.WriteLine("Task is faulted or canceled (exception) - {0}", passed ? "Passed" : "FAILED"); } return passed; }
/// <summary> /// Create a logging thread to process the logging queue /// </summary> private void CreateLoggingEventQueue() { // We are creating a two-node dataflow graph here. The first node is a buffer, which will hold up to the number of // logging events we have specified as the queueCapacity. The second node is the processor which will actually process each message. // When the capacity of the buffer is reached, further attempts to send messages to it will block. // The reason we can't just set the BoundedCapacity on the processing block is that ActionBlock has some weird behavior // when the queue capacity is reached. Specifically, it will block new messages from being processed until it has // entirely drained its input queue, as opposed to letting new ones in as old ones are processed. This is logged as // a perf bug (305575) against Dataflow. If they choose to fix it, we can eliminate the buffer node from the graph. var dataBlockOptions = new DataflowBlockOptions { BoundedCapacity = Convert.ToInt32(_queueCapacity) }; _loggingQueue = new BufferBlock<object>(dataBlockOptions); var executionDataBlockOptions = new ExecutionDataflowBlockOptions { BoundedCapacity = 1 }; _loggingQueueProcessor = new ActionBlock<object>(loggingEvent => LoggingEventProcessor(loggingEvent), executionDataBlockOptions); var dataLinkOptions = new DataflowLinkOptions { PropagateCompletion = true }; _loggingQueue.LinkTo(_loggingQueueProcessor, dataLinkOptions); }
private void Initialize( Action <KeyValuePair <TInput, long> > processMessageAction, ExecutionDataflowBlockOptions dataflowBlockOptions, [NotNull] ref SourceCore <TOutput>?source, [NotNull] ref TargetCore <TInput>?target, ref ReorderingBuffer <IEnumerable <TOutput> >?reorderingBuffer, TargetCoreOptions targetCoreOptions) { if (dataflowBlockOptions == null) { throw new ArgumentNullException(nameof(dataflowBlockOptions)); } // Ensure we have options that can't be changed by the caller dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone(); // Initialize onItemsRemoved delegate if necessary Action <ISourceBlock <TOutput>, int>?onItemsRemoved = null; if (dataflowBlockOptions.BoundedCapacity > 0) { onItemsRemoved = (owningSource, count) => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.ChangeBoundingCount(-count); } // Initialize source component source = new SourceCore <TOutput>(this, dataflowBlockOptions, owningSource => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.Complete(exception: null, dropPendingMessages: true), onItemsRemoved); // If parallelism is employed, we will need to support reordering messages that complete out-of-order. // However, a developer can override this with EnsureOrdered == false. if (dataflowBlockOptions.SupportsParallelExecution && dataflowBlockOptions.EnsureOrdered) { reorderingBuffer = new ReorderingBuffer <IEnumerable <TOutput> >( this, (source, messages) => ((TransformManyBlock <TInput, TOutput>)source)._source.AddMessages(messages)); } // Create the underlying target and source target = new TargetCore <TInput>(this, processMessageAction, _reorderingBuffer, dataflowBlockOptions, targetCoreOptions); // Link up the target half with the source half. In doing so, // ensure exceptions are propagated, and let the source know no more messages will arrive. // As the target has completed, and as the target synchronously pushes work // through the reordering buffer when async processing completes, // we know for certain that no more messages will need to be sent to the source. target.Completion.ContinueWith((completed, state) => { var sourceCore = (SourceCore <TOutput>)state !; if (completed.IsFaulted) { sourceCore.AddAndUnwrapAggregateException(completed.Exception !); } sourceCore.Complete(); }, source, CancellationToken.None, Common.GetContinuationOptions(), TaskScheduler.Default); // It is possible that the source half may fault on its own, e.g. due to a task scheduler exception. // In those cases we need to fault the target half to drop its buffered messages and to release its // reservations. This should not create an infinite loop, because all our implementations are designed // to handle multiple completion requests and to carry over only one. source.Completion.ContinueWith((completed, state) => { var thisBlock = ((TransformManyBlock <TInput, TOutput>)state !) as IDataflowBlock; Debug.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion."); thisBlock.Fault(completed.Exception !); }, this, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default); // Handle async cancellation requests by declining on the target Common.WireCancellationToComplete( dataflowBlockOptions.CancellationToken, Completion, state => ((TargetCore <TInput>)state !).Complete(exception: null, dropPendingMessages: true), target); DataflowEtwProvider etwLog = DataflowEtwProvider.Log; if (etwLog.IsEnabled()) { etwLog.DataflowBlockCreated(this, dataflowBlockOptions); } }
/// <summary>Initializes the <see cref="TransformManyBlock{TInput,TOutput}"/> with the specified function and <see cref="ExecutionDataflowBlockOptions"/>.</summary> /// <param name="transform"> /// The function to invoke with each data element received. All of the data from the returned in the <see cref="System.Collections.Generic.IEnumerable{TOutput}"/> /// will be made available as output from this <see cref="TransformManyBlock{TInput,TOutput}"/>. /// </param> /// <param name="dataflowBlockOptions">The options with which to configure this <see cref="TransformManyBlock{TInput,TOutput}"/>.</param> /// <exception cref="System.ArgumentNullException">The <paramref name="transform"/> is null (Nothing in Visual Basic).</exception> /// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception> public TransformManyBlock(Func <TInput, IEnumerable <TOutput> > transform, ExecutionDataflowBlockOptions dataflowBlockOptions) : this(transform, null, dataflowBlockOptions) { }