public void Run()
{
this.output.WriteLine("TPL start");
this.CheckFileExists();
var f = new FileInfo(this.sampleLogFileName);
this.fileSizeInBytes = f.Length;
this.lineSizeInBytesSoFar = 0;
this.lineCount = 0;
var sw = new Stopwatch();
sw.Start();
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 8 };
var ab = new ActionBlock<string>(s => this.ProcessLine(s), options);
using (TextReader reader = File.OpenText(this.sampleLogFileName))
{
string line;
while ((line = reader.ReadLine()) != null)
{
ab.Post(line);
}
}
ab.Complete();
ab.Completion.Wait();
sw.Stop();
this.ShowResults();
this.output.WriteLine();
this.output.WriteLine("TPL done in {0}", sw.Elapsed);
}
/// <summary>
/// Creates the Disk output action block.
/// </summary>
/// <param name="containerName">Name of the container.</param>
/// <param name="extension">The extension.</param>
/// <param name="context">The context.</param>
/// <param name="options">The options.</param>
/// <returns>ActionBlock<System.Byte[]>.</returns>
public static ActionBlock<byte[]> Create(CloudStorageAccount account, string containerName, string extension, string partitionId, ExecutionDataflowBlockOptions options)
{
var container = GetOrCreateContainer(account, containerName);
Logger.Information("Creating appender for {Partition} using blob container {Container}", partitionId, container.Uri.AbsoluteUri);
return new ActionBlock<byte[]>(async data =>
{
if (data.Length > 4194304)
{
Logger.Fatal("Maximum size for blob append is 4MB, buffer size is {BufferSize}", data.Length);
return;
}
try
{
var blob = GetAppendBlob(container, extension, partitionId);
Logger.Debug("Appending {Length} bytes to blob {Uri}", data.Length, blob.Uri.AbsoluteUri);
using (Logger.BeginTimedOperation("AppendFromByteArrayAsync", blob.Uri.AbsoluteUri, LogEventLevel.Debug))
await blob.AppendFromByteArrayAsync(data, 0, data.Length);
}
catch (Exception ex)
{
Logger.Error(ex, "BlobAppender Append From Byte Array Partition " + partitionId);
}
}, options);
}
public DataflowWebHookSenderTests()
{
_handlerMock = new HttpMessageHandlerMock();
_httpClient = new HttpClient(_handlerMock);
_options = new ExecutionDataflowBlockOptions();
_loggerMock = new Mock<ILogger>();
}
public void Start()
{
var sink = new ActionBlock<PageResultMessage>((Action<PageResultMessage>)Sink);
var source = new BufferBlock<GetPageMessage>();
var linkOptions = new DataflowLinkOptions {PropagateCompletion = false};
for (int i = 0; i < 10; i++)
{
var options = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1
};
var worker = new TransformBlock<GetPageMessage, PageResultMessage>(
(Func<GetPageMessage, PageResultMessage>)Worker, options);
source.LinkTo(worker, linkOptions);
worker.LinkTo(sink, linkOptions);
}
foreach (var url in UrlList.Urls)
{
source.Post(new GetPageMessage{ Url = url });
}
source.Complete();
sink.Completion.Wait();
}
public void ProduceLogs(int count, int buffSize)
{
var bufferOptions = new DataflowBlockOptions() { BoundedCapacity = buffSize };
var writerOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 10, MaxDegreeOfParallelism = 1, MaxMessagesPerTask = 10, SingleProducerConstrained = true };
LogGenerator g = new LogGenerator();
var file = new StreamWriter("basic.async.buff.log", false);
BufferBlock<string> buffer = new BufferBlock<string>(bufferOptions);
ActionBlock<string> writer = new ActionBlock<string>(s => file.WriteLine(s), writerOptions);
buffer.LinkTo(writer, new DataflowLinkOptions() { PropagateCompletion = true });
for (int i = 0; i < count; i++)
{
g.Next();
var line = string.Format(g.FormatStr, g.Param1, g.Param2, g.Param3, g.Param4, g.Param5, g.Param6);
writer.SendAsync(line).Wait();
}
buffer.Complete();
Completed = writer.Completion.ContinueWith(t => file.Close());
}
/// <summary>
/// Initialize a new instance of the <see cref="DataflowWebHookSender"/> with the given retry policy, <paramref name="options"/>,
/// and <paramref name="httpClient"/>. This constructor is intended for unit testing purposes.
/// </summary>
internal DataflowWebHookSender(
ILogger logger,
IEnumerable<TimeSpan> retryDelays,
ExecutionDataflowBlockOptions options,
HttpClient httpClient,
Action<WebHookWorkItem> onWebHookSuccess,
Action<WebHookWorkItem> onWebHookFailure)
: base(logger)
{
retryDelays = retryDelays ?? DefaultRetries;
options = options ?? new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = DefaultMaxConcurrencyLevel
};
_httpClient = httpClient ?? new HttpClient();
// Create the launch processors with the given retry delays
_launchers = new ActionBlock<WebHookWorkItem>[1 + retryDelays.Count()];
int offset = 0;
_launchers[offset++] = new ActionBlock<WebHookWorkItem>(async item => await LaunchWebHook(item), options);
foreach (TimeSpan delay in retryDelays)
{
_launchers[offset++] = new ActionBlock<WebHookWorkItem>(async item => await DelayedLaunchWebHook(item, delay));
}
string msg = string.Format(CultureInfo.CurrentCulture, CustomResources.Manager_Started, typeof(DataflowWebHookSender).Name, _launchers.Length);
Logger.Info(msg);
// Set handlers for testing purposes
_onWebHookSuccess = onWebHookSuccess;
_onWebHookFailure = onWebHookFailure;
}
public static async Task <ActionBlock <T> > ParallelEnumerateAsync <T>(IEnumerable <T> items, CancellationToken cancellationToken, int maxDegreeOfParallelism, Func <T, CancellationToken, Task> func)
{
var dataflowBlockOptions = new System.Threading.Tasks.Dataflow.ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = maxDegreeOfParallelism,
CancellationToken = cancellationToken
};
var workerBlock = new ActionBlock <T>(item => func(item, cancellationToken), dataflowBlockOptions);
foreach (var item in items)
{
await Policy
.Handle <Exception>()
.WaitAndRetryAsync(new[] { TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(60) })
.ExecuteAsync(async(ct) =>
{
var sent = await workerBlock.SendAsync(item, cancellationToken);
if (!sent)
{
throw new Exception($"Could not process item: {item}");
}
},
cancellationToken
);
}
workerBlock.Complete();
await workerBlock.Completion;
return(workerBlock);
}
public static TransformBlock<EventData, byte[]> Create(ExecutionDataflowBlockOptions options)
{
return new TransformBlock<EventData, byte[]>(data =>
{
return data.GetBytes();
}, options);
}
public void MagnesiumInit()
{
_dcpManager = new DcpManager();
_requestQueueOptions = new ExecutionDataflowBlockOptions();
_storeQueue = new ActionBlock<DataContainer>(dc => _dcpManager.StoreHandler(dc));
_parseQueue = new ActionBlock<DataResponse>(dresp => _dcpManager.ParseHandler(dresp, _storeQueue));
_requestQueue = new ActionBlock<DataRequest>(dr => _dcpManager.RequestHandler(dr, _parseQueue), _requestQueueOptions);
//布隆过滤器, 大小10M, 容错率0.001
_bloomFilter = new BloomFilter<string>(10000000, 0.001f);
}
/// <summary>
/// Creates the event sender action block.
/// </summary>
/// <param name="hubClient">The hub client.</param>
/// <param name="options">The options.</param>
/// <returns>ActionBlock<System.Byte[]>.</returns>
public static ActionBlock<byte[]> Create(EventHubClient hubClient, ExecutionDataflowBlockOptions options)
{
return new ActionBlock<byte[]>(data =>
{
//var index = Array.IndexOf(data, (byte)'>');
//if (index < 0) return Task.FromResult(0);
//var key = Encoding.UTF8.GetString(data, 0, index);
var eventData = new EventData(data);
return hubClient.SendAsync(eventData);
}, options);
}
public WebHookManagerTests()
{
_handlerMock = new HttpMessageHandlerMock();
_httpClient = new HttpClient(_handlerMock);
_storeMock = new Mock<IWebHookStore>();
_storeMock.Setup<Task<ICollection<WebHook>>>(s => s.QueryWebHooksAsync(TestUser, new[] { "a1" }))
.ReturnsAsync(new Collection<WebHook> { CreateWebHook() })
.Verifiable();
_options = new ExecutionDataflowBlockOptions();
_loggerMock = new Mock<ILogger>();
_response = new HttpResponseMessage();
}
/// <summary>
/// Initializes a new instance of the <see cref="ThreadedLogger" /> class.
/// </summary>
/// <param name="log">The log.</param>
public ThreadedLogger(Action<string> log)
{
Guard.NotNull(log);
_CancellationTokenSource = new CancellationTokenSource();
var options = new ExecutionDataflowBlockOptions
{
CancellationToken = _CancellationTokenSource.Token,
MaxDegreeOfParallelism = 1
};
_ActionBlock = new ActionBlock<string>(log, options);
}
public AggregationResult(EntityMetadata metadata, AggregationConfig config, int keyIndex)
{
_metadata = metadata;
_config = config;
_keyIndex = keyIndex;
_aggregators = config.Aggregators.Select(c => c.CreateAggregator()).ToArray();
_groupResults = keyIndex < metadata.KeyCount ? new ConcurrentDictionary<int, AggregationResult>() : null;
_cts = new CancellationTokenSource();
var executionOptions = new ExecutionDataflowBlockOptions {
CancellationToken = _cts.Token,
SingleProducerConstrained = true
};
_changeBlock = new ActionBlock<AggregationChange>((Action<AggregationChange>)ProcessChange, executionOptions);
}
static IEnumerable<int[]> GetTaskIdsForExecutionsOptions (
ExecutionDataflowBlockOptions options)
{
var blockFactories =
new Func<ConcurrentQueue<Tuple<int, int>>, ITargetBlock<int>>[]
{
q => new ActionBlock<int> (
i => q.Enqueue (Tuple.Create (i, Task.CurrentId.Value)), options),
q => new TransformBlock<int, int> (i =>
{
q.Enqueue (Tuple.Create (i, Task.CurrentId.Value));
return i;
}, options),
q => new TransformManyBlock<int, int> (i =>
{
q.Enqueue (Tuple.Create (i, Task.CurrentId.Value));
return new[] { i };
}, options)
};
foreach (var factory in blockFactories) {
var queue = new ConcurrentQueue<Tuple<int, int>> ();
var block = factory (queue);
Assert.IsEmpty (queue);
for (int i = 0; i < 100; i++)
block.Post (i);
block.Complete ();
var source = block as ISourceBlock<int>;
if (source != null) {
Assert.IsFalse (block.Completion.Wait (100));
source.LinkTo (new BufferBlock<int> ());
}
Assert.IsTrue (block.Completion.Wait (1000));
CollectionAssert.AreEquivalent (
Enumerable.Range (0, 100), queue.Select (t => t.Item1));
yield return queue.Select (t => t.Item2).ToArray ();
}
}
/// <summary>
/// Creates the batch.
/// </summary>
/// <param name="hubClient">The hub client.</param>
/// <param name="options">The options.</param>
/// <returns>ActionBlock<IEnumerable<System.Byte[]>>.</returns>
public static ActionBlock<IEnumerable<byte[]>> CreateBatch(EventHubClient hubClient, ExecutionDataflowBlockOptions options)
{
return new ActionBlock<IEnumerable<byte[]>>(async batch =>
{
var size = 0;
var output = batch.Select(data => {
//var index = Array.IndexOf(data, (byte)'>');
//var key = index < 0 ? null : Encoding.UTF8.GetString(data, 0, index);
size += data.Length;
return new EventData(data);// { PartitionKey = key };
}).ToList();
Logger.Debug("Sending {Count} events ({Size} bytes)", output.Count, size);
using (Logger.BeginTimedOperation("SendBatch (" + output.Count + ")", null, LogEventLevel.Debug))
await Task.WhenAll(output.GroupBy(x => x.PartitionKey).Select(x => hubClient.SendPartitionedBatchAsync(x, true)));
}, options);
}
public static void Example1()
{
var conf = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 4 };
ActionBlock<int> a = new ActionBlock<int>(i =>
{
Thread.Sleep(500);
Console.WriteLine(i);
}, conf);
TransformBlock<int, int> t = new TransformBlock<int, int>(i => i * 3);
t.LinkTo(a);
for (int i = 0; i < 12; i++)
{
t.Post(i);
}
}
public void Run()
{
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 8 };
var tb = new TransformBlock<int, int>(i => i * 2);
var ab = new ActionBlock<int>(i => this.Compute(i), options);
tb.LinkTo(ab);
for (var i = 0; i < 10; i++)
{
tb.Post(i);
}
tb.Complete();
tb.Completion.Wait();
Thread.Sleep(500);
}
static IEnumerable<ITargetBlock<int>> GetExecutionBlocksWithAsyncAction (
Func<int, Task> action, ExecutionDataflowBlockOptions options)
{
yield return new ActionBlock<int> (action, options);
yield return new TransformBlock<int, int> (
i => action (i).ContinueWith (
t =>
{
t.Wait ();
return i;
}), options);
yield return new TransformManyBlock<int, int> (
i => action (i).ContinueWith (
t =>
{
t.Wait ();
return Enumerable.Empty<int> ();
}), options);
}
/// <summary>
/// Creates the Disk output action block. Returns length of data written.
/// </summary>
/// <param name="filePath">The file path.</param>
/// <param name="extension">The extension.</param>
/// <param name="partitionId">The partition identifier.</param>
/// <param name="options">The options.</param>
/// <returns>ActionBlock<System.Byte[]>.</returns>
public static TransformBlock<byte[], long> Create(string filePath, string extension, string partitionId, ExecutionDataflowBlockOptions options)
{
return new TransformBlock<byte[], long>(async data =>
{
var fileName = Path.Combine(filePath, string.Concat(DateTime.UtcNow.ToString("yyyyMMddHH"), 'p', partitionId, '.', extension));
Logger.Debug("Appending {Length} bytes to file {Uri}", data.Length, fileName);
using (Logger.BeginTimedOperation("WriteAsync", fileName, LogEventLevel.Debug))
using (var outputStream = File.OpenWrite(fileName))
{
outputStream.Position = outputStream.Length;
await outputStream.WriteAsync(data, 0, data.Length);
await outputStream.FlushAsync();
}
return data.Length;
}, options);
}
public async Task StartAsync(CancellationToken cancellationToken)
{
ThrowIfDisposed();
if (_watcher != null && _watcher.EnableRaisingEvents)
{
throw new InvalidOperationException("The listener has already been started.");
}
if (string.IsNullOrEmpty(_config.RootPath) || !Directory.Exists(_config.RootPath))
{
throw new InvalidOperationException(string.Format("Path '{0}' is invalid. FilesConfiguration.RootPath must be set to a valid directory location.", _config.RootPath));
}
CreateFileWatcher();
FileProcessorFactoryContext context = new FileProcessorFactoryContext(_config, _attribute, _triggerExecutor, _trace);
_processor = _config.ProcessorFactory.CreateFileProcessor(context);
ExecutionDataflowBlockOptions options = new ExecutionDataflowBlockOptions
{
BoundedCapacity = _processor.MaxQueueSize,
MaxDegreeOfParallelism = _processor.MaxDegreeOfParallelism,
};
_workQueue = new ActionBlock<FileSystemEventArgs>(async (e) => await ProcessWorkItem(e), options);
// on startup, process any preexisting files that haven't been processed yet
ProcessFiles();
// Create a timer to cleanup processed files.
// The timer doesn't auto-reset. It will reset itself
// when we receive more file events
_cleanupTimer = new System.Timers.Timer()
{
AutoReset = false,
Interval = _rand.Next(5 * 1000, 8 * 1000)
};
_cleanupTimer.Elapsed += OnCleanupTimer;
_cleanupTimer.Start();
await Task.FromResult<bool>(true);
}
/// <summary>
/// 初期化処理。
/// </summary>
/// <param name="settingObject"></param>
/// <param name="token"></param>
/// <returns></returns>
public void Init(dynamic settingObject, CancellationToken token)
{
logger.Trace("Init Start");
token.Register(Dispose);
var opt = new DataflowBlockOptions
{
CancellationToken = token,
};
var buffer = new BufferBlock<PastaLog>(opt);
var actOpt = new ExecutionDataflowBlockOptions()
{
CancellationToken = token,
SingleProducerConstrained = true,
};
var act = new ActionBlock<PastaLog>((a) => Save(a), actOpt);
buffer.LinkTo(act);
Target = buffer;
logger.Trace("Init End");
}
public static void Example2()
{
// Scenario: one buffer and 3 actions connected to it - each number will be processed only by one action
var conf = new ExecutionDataflowBlockOptions();
// todo: play with those
conf.BoundedCapacity = 1;
conf.MaxDegreeOfParallelism = 4;
var buffer = new BufferBlock<int>();
var action1 = new ActionBlock<int>(a => {
Thread.Sleep(50);
Console.WriteLine("Action 1 value: {0}", a);
}, conf);
var action2 = new ActionBlock<int>(a =>
{
Thread.Sleep(50);
Console.WriteLine("Action 2 value: {0}", a);
}, conf);
var action3 = new ActionBlock<int>(a =>
{
Thread.Sleep(50);
Console.WriteLine("Action 3 value: {0}", a);
}, conf);
buffer.LinkTo(action1);
buffer.LinkTo(action2);
buffer.LinkTo(action3);
var t = new Task(() => {
for (int i = 0; i < 12; i++)
{
buffer.Post(i);
}
});
t.Start();
}
public static void Example3()
{
// Scenario: one buffer and 3 actions connected to it - each number precessed is send to each action
var conf = new ExecutionDataflowBlockOptions();
//conf.MaxDegreeOfParallelism = 4;
var buffer = new BroadcastBlock<int>(i => i);
var action1 = new ActionBlock<int>(a =>
{
Thread.Sleep(50);
Console.WriteLine("Action 1 value: {0}", a);
}, conf);
var action2 = new ActionBlock<int>(a =>
{
Thread.Sleep(50);
Console.WriteLine("Action 2 value: {0}", a);
}, conf);
var action3 = new ActionBlock<int>(a =>
{
Thread.Sleep(50);
Console.WriteLine("Action 3 value: {0}", a);
}, conf);
buffer.LinkTo(action1);
buffer.LinkTo(action2);
buffer.LinkTo(action3);
var t = new Task(() =>
{
for (int i = 0; i < 12; i++)
{
buffer.Post(i);
}
});
t.Start();
}
/// <summary>
/// Instantiate a message processor, using the provided handlers
/// and timeout/concurrency configuration
/// </summary>
public EventProcessor(IMessageHandlerResolver handlerResolver,
ICircuitBreaker circuitBreaker,
int maxConcurrency,
string eventHubName,
IDispatcherInstrumentationPublisher instrumentationPublisher)
{
Logger.Info(
"Initializing event processor with concurrency {0}",
maxConcurrency);
_handlerResolver = handlerResolver;
_circuitBreaker = circuitBreaker;
_eventHubName = eventHubName;
_instrumentationPublisher = instrumentationPublisher;
// Set up the execution options for bounded concurrency
// when using the TPL data flow action block
_options = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = maxConcurrency
};
}
public void Run()
{
var propagateCompletion = new DataflowLinkOptions { PropagateCompletion = true };
var csep = new ConcurrentExclusiveSchedulerPair();
var concurrentOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 2,
TaskScheduler = csep.ConcurrentScheduler
};
var exclusiveOptions = new ExecutionDataflowBlockOptions
{
TaskScheduler = csep.ExclusiveScheduler
};
var concurrent = new ActionBlock<int>(async i =>
{
Console.WriteLine("Concurent print value: {0}", i);
await Task.Delay(500);
}, concurrentOptions);
var exclusive = new ActionBlock<int>(i =>
{
Console.WriteLine("Exclusive print value: {0}", i);
Thread.Sleep(750);
}, exclusiveOptions);
var broadcaster = new BroadcastBlock<int>(i => i);
broadcaster.LinkTo(concurrent, propagateCompletion);
broadcaster.LinkTo(exclusive, propagateCompletion, i => i % 2 == 0);
Enumerable
.Range(1, 10)
.ToList()
.ForEach(i => broadcaster.Post(i));
broadcaster.Complete();
Task.WaitAll(concurrent.Completion, exclusive.Completion);
}
/// <summary>
/// Creates the Transformation Block to BZip2 compress.
/// </summary>
/// <param name="options">The options.</param>
public static TransformBlock<byte[], byte[]> Create(ExecutionDataflowBlockOptions options)
{
return new TransformBlock<byte[], byte[]>(data =>
{
using (var output = new MemoryStream())
{
using (Logger.BeginTimedOperation("BZip2OutputStream", null, LogEventLevel.Debug))
using (var input = new MemoryStream(data))
using (var compression = new BZip2OutputStream(output))
{
input.CopyTo(compression);
compression.Flush();
}
var result = output.ToArray();
Logger.Debug("Compressed {InputSize} bytes to {OutputSize} ({Compression:00}%)", data.Length, result.Length, result.Length * 100M/data.Length);
return result;
}
}, options);
}
public void MaxDegreeOfParallelismTest()
{
// loop to better test for race conditions
// some that showed in this test were quite rare
for (int i = 0; i < 10; i++)
{
var options = new ExecutionDataflowBlockOptions ();
foreach (var taskIds in GetTaskIdsForExecutionsOptions(options))
Assert.AreEqual (1, CalculateDegreeOfParallelism (taskIds));
options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 };
foreach (var taskIds in GetTaskIdsForExecutionsOptions (options))
Assert.LessOrEqual (CalculateDegreeOfParallelism (taskIds), 2);
options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 4 };
foreach (var taskIds in GetTaskIdsForExecutionsOptions (options))
Assert.LessOrEqual (CalculateDegreeOfParallelism (taskIds), 4);
options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = -1 };
foreach (var taskIds in GetTaskIdsForExecutionsOptions (options))
Assert.LessOrEqual (CalculateDegreeOfParallelism (taskIds), taskIds.Length);
}
}
/// <summary>
/// Initializes the <see cref="TransformBlock{TInput,TOutput}"/> with the specified <see cref="System.Func{TInput,TOutput}"/>
/// and <see cref="DataflowBlockOptions"/>.
/// </summary>
/// <param name="transformSync">The synchronous function to invoke with each data element received.</param>
/// <param name="transformAsync">The asynchronous function to invoke with each data element received.</param>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="TransformBlock{TInput,TOutput}"/>.</param>
/// <exception cref="System.ArgumentNullException">The <paramref name="transformSync"/> and <paramref name="transformAsync"/> are both null (Nothing in Visual Basic).</exception>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
private TransformBlock(Func <TInput, TOutput> transformSync, Func <TInput, Task <TOutput> > transformAsync, ExecutionDataflowBlockOptions dataflowBlockOptions)
{
if (transformSync == null && transformAsync == null)
{
throw new ArgumentNullException("transform");
}
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException("dataflowBlockOptions");
}
Contract.Requires(transformSync == null ^ transformAsync == null, "Exactly one of transformSync and transformAsync must be null.");
Contract.EndContractBlock();
// Ensure we have options that can't be changed by the caller
dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone();
// Initialize onItemsRemoved delegate if necessary
Action <ISourceBlock <TOutput>, int> onItemsRemoved = null;
if (dataflowBlockOptions.BoundedCapacity > 0)
{
onItemsRemoved = (owningSource, count) => ((TransformBlock <TInput, TOutput>)owningSource)._target.ChangeBoundingCount(-count);
}
// Initialize source component.
_source = new SourceCore <TOutput>(this, dataflowBlockOptions,
owningSource => ((TransformBlock <TInput, TOutput>)owningSource)._target.Complete(exception: null, dropPendingMessages: true),
onItemsRemoved);
// If parallelism is employed, we will need to support reordering messages that complete out-of-order
if (dataflowBlockOptions.SupportsParallelExecution)
{
_reorderingBuffer = new ReorderingBuffer <TOutput>(this, (owningSource, message) => ((TransformBlock <TInput, TOutput>)owningSource)._source.AddMessage(message));
}
// Create the underlying target
if (transformSync != null) // sync
{
_target = new TargetCore <TInput>(this,
messageWithId => ProcessMessage(transformSync, messageWithId),
_reorderingBuffer, dataflowBlockOptions, TargetCoreOptions.None);
}
else // async
{
Contract.Assert(transformAsync != null, "Incorrect delegate type.");
_target = new TargetCore <TInput>(this,
messageWithId => ProcessMessageWithTask(transformAsync, messageWithId),
_reorderingBuffer, dataflowBlockOptions, TargetCoreOptions.UsesAsyncCompletion);
}
// Link up the target half with the source half. In doing so,
// ensure exceptions are propagated, and let the source know no more messages will arrive.
// As the target has completed, and as the target synchronously pushes work
// through the reordering buffer when async processing completes,
// we know for certain that no more messages will need to be sent to the source.
#if PRENET45
m_target.Completion.ContinueWith(completed =>
{
if (completed.IsFaulted)
{
m_source.AddAndUnwrapAggregateException(completed.Exception);
}
m_source.Complete();
}, CancellationToken.None, Common.GetContinuationOptions(), TaskScheduler.Default);
#else
_target.Completion.ContinueWith((completed, state) =>
{
var sourceCore = (SourceCore <TOutput>)state;
if (completed.IsFaulted)
{
sourceCore.AddAndUnwrapAggregateException(completed.Exception);
}
sourceCore.Complete();
}, _source, CancellationToken.None, Common.GetContinuationOptions(), TaskScheduler.Default);
#endif
// It is possible that the source half may fault on its own, e.g. due to a task scheduler exception.
// In those cases we need to fault the target half to drop its buffered messages and to release its
// reservations. This should not create an infinite loop, because all our implementations are designed
// to handle multiple completion requests and to carry over only one.
#if PRENET45
m_source.Completion.ContinueWith(completed =>
{
Contract.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion.");
(this as IDataflowBlock).Fault(completed.Exception);
}, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default);
#else
_source.Completion.ContinueWith((completed, state) =>
{
var thisBlock = ((TransformBlock <TInput, TOutput>)state) as IDataflowBlock;
Contract.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion.");
thisBlock.Fault(completed.Exception);
}, this, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default);
#endif
// Handle async cancellation requests by declining on the target
Common.WireCancellationToComplete(
dataflowBlockOptions.CancellationToken, Completion, state => ((TargetCore <TInput>)state).Complete(exception: null, dropPendingMessages: true), _target);
#if FEATURE_TRACING
var etwLog = DataflowEtwProvider.Log;
if (etwLog.IsEnabled())
{
etwLog.DataflowBlockCreated(this, dataflowBlockOptions);
}
#endif
}
/// <summary>Initializes the <see cref="TransformManyBlock{TInput,TOutput}"/> with the specified function and <see cref="ExecutionDataflowBlockOptions"/>.</summary>
/// <param name="transform">
/// The function to invoke with each data element received. All of the data asynchronously returned in the <see cref="System.Collections.Generic.IEnumerable{TOutput}"/>
/// will be made available as output from this <see cref="TransformManyBlock{TInput,TOutput}"/>.
/// </param>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="TransformManyBlock{TInput,TOutput}"/>.</param>
/// <exception cref="System.ArgumentNullException">The <paramref name="transform"/> is null (Nothing in Visual Basic).</exception>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
public TransformManyBlock(Func <TInput, Task <IEnumerable <TOutput> > > transform, ExecutionDataflowBlockOptions dataflowBlockOptions)
{
if (transform == null)
{
throw new ArgumentNullException(nameof(transform));
}
Initialize(messageWithId => ProcessMessageWithTask(transform, messageWithId), dataflowBlockOptions, ref _source, ref _target, ref _reorderingBuffer, TargetCoreOptions.UsesAsyncCompletion);
}
//[Fact(Skip = "Outerloop")]
public void RunTransformBlockConformanceTests()
{
bool passed = true;
// SYNC
#region Sync
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> transformBlockFactory =
options =>
{
TransformBlock<int, int> transformBlock = new TransformBlock<int, int>(i => i, (ExecutionDataflowBlockOptions)options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = transformBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2, CancellationToken = cancellationSource.Token };
passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, dopOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 1, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, cancellationOptions, 1, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2));
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16;
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformBlock<int, int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return i;
});
for (int i = 0; i < 2; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 2; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test using a precanceled token
{
bool localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token };
var t = new TransformBlock<int, int>(i => i, dbo);
int ignoredValue;
IList<int> ignoredValues;
localPassed &= t.LinkTo(new ActionBlock<int>(delegate { })) != null;
localPassed &= t.SendAsync(42).Result == false;
localPassed &= t.TryReceiveAll(out ignoredValues) == false;
localPassed &= t.Post(42) == false;
localPassed &= t.OutputCount == 0;
localPassed &= t.TryReceive(out ignoredValue) == false;
localPassed &= t.Completion != null;
t.Complete();
}
catch (Exception)
{
localPassed = false;
}
Console.WriteLine(" {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting
{
bool localPassed = true;
var t = new TransformBlock<int, int>(new Func<int, int>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" {0}: Faulted handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
#region Async
// ASYNC (a copy of the sync but with constructors returning Task<T> instead of T
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> transformBlockFactory =
options =>
{
TransformBlock<int, int> transformBlock = new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i), (ExecutionDataflowBlockOptions)options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = transformBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2, CancellationToken = cancellationSource.Token };
passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, dopOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2)));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2)));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2)));
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16;
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2)));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformBlock<int, int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return Task.Factory.StartNew(() => i);
});
for (int i = 0; i < 2; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 2; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored
{
bool localPassed = true;
var t = new TransformBlock<int, int>(i =>
{
if ((i % 2) == 0) return null;
return Task.Factory.StartNew(() => i);
});
for (int i = 0; i < 2; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 2; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: null tasks are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored when a reordering buffer is in place
{
bool localPassed = true;
var t = new TransformBlock<int, int>(i =>
{
if (i == 0)
{
Task.Delay(10).Wait();
return null;
}
return Task.Factory.StartNew(() => i);
}, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 });
t.Post(0);
t.Post(1);
try
{
localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1;
}
catch
{
localPassed = false;
}
Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the delegate
{
bool localPassed = true;
var t = new TransformBlock<int, int>(new Func<int, Task<int>>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the task
{
bool localPassed = true;
var t = new TransformBlock<int, int>(new Func<int, Task<int>>(i => Task<int>.Factory.StartNew(() => { throw new InvalidOperationException(); })));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
Assert.True(passed, "Test failed.");
}
/// <summary>Initializes the <see cref="ActionBlock{T}"/> with the specified <see cref="System.Func{T,Task}"/> and <see cref="ExecutionDataflowBlockOptions"/>.</summary>
/// <param name="action">The action to invoke with each data element received.</param>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="ActionBlock{T}"/>.</param>
/// <exception cref="System.ArgumentNullException">The <paramref name="action"/> is null (Nothing in Visual Basic).</exception>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
public ActionBlock(Func <TInput, Task> action, ExecutionDataflowBlockOptions dataflowBlockOptions) :
this((Delegate)action, dataflowBlockOptions)
{
}
/// <summary>Initializes the <see cref="ActionBlock{T}"/> with the specified delegate and options.</summary>
/// <param name="action">The action to invoke with each data element received.</param>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="ActionBlock{T}"/>.</param>
/// <exception cref="System.ArgumentNullException">The <paramref name="action"/> is null (Nothing in Visual Basic).</exception>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
private ActionBlock(Delegate action, ExecutionDataflowBlockOptions dataflowBlockOptions)
{
// Validate arguments
if (action == null)
{
throw new ArgumentNullException(nameof(action));
}
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException(nameof(dataflowBlockOptions));
}
// Ensure we have options that can't be changed by the caller
dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone();
// Based on the mode, initialize the target. If the user specifies SingleProducerConstrained,
// we'll try to employ an optimized mode under a limited set of circumstances.
var syncAction = action as Action <TInput>;
if (syncAction != null &&
dataflowBlockOptions.SingleProducerConstrained &&
dataflowBlockOptions.MaxDegreeOfParallelism == 1 &&
!dataflowBlockOptions.CancellationToken.CanBeCanceled &&
dataflowBlockOptions.BoundedCapacity == DataflowBlockOptions.Unbounded)
{
// Initialize the SPSC fast target to handle the bulk of the processing.
// The SpscTargetCore is only supported when BoundedCapacity, CancellationToken,
// and MaxDOP are all their default values. It's also only supported for sync
// delegates and not for async delegates.
_spscTarget = new SpscTargetCore <TInput>(this, syncAction, dataflowBlockOptions);
}
else
{
// Initialize the TargetCore which handles the bulk of the processing.
// The default target core can handle all options and delegate flavors.
if (syncAction != null) // sync
{
_defaultTarget = new TargetCore <TInput>(this,
messageWithId => ProcessMessage(syncAction, messageWithId),
null, dataflowBlockOptions, TargetCoreOptions.RepresentsBlockCompletion);
}
else // async
{
var asyncAction = action as Func <TInput, Task>;
Debug.Assert(asyncAction != null, "action is of incorrect delegate type");
_defaultTarget = new TargetCore <TInput>(this,
messageWithId => ProcessMessageWithTask(asyncAction, messageWithId),
null, dataflowBlockOptions, TargetCoreOptions.RepresentsBlockCompletion | TargetCoreOptions.UsesAsyncCompletion);
}
// Handle async cancellation requests by declining on the target
Common.WireCancellationToComplete(
dataflowBlockOptions.CancellationToken, Completion, state => ((TargetCore <TInput>)state).Complete(exception: null, dropPendingMessages: true), _defaultTarget);
}
#if FEATURE_TRACING
DataflowEtwProvider etwLog = DataflowEtwProvider.Log;
if (etwLog.IsEnabled())
{
etwLog.DataflowBlockCreated(this, dataflowBlockOptions);
}
#endif
Debug.Assert((_spscTarget != null) ^ (_defaultTarget != null), "One and only one of the two targets must be non-null after construction");
}
/// <summary>
/// Initializes the <see cref="TransformBlock{TInput,TOutput}"/> with the specified <see cref="System.Func{TInput,TOutput}"/>
/// and <see cref="ExecutionDataflowBlockOptions"/>.
/// </summary>
/// <param name="transform">The function to invoke with each data element received.</param>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="TransformBlock{TInput,TOutput}"/>.</param>
/// <exception cref="System.ArgumentNullException">The <paramref name="transform"/> is null (Nothing in Visual Basic).</exception>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
public TransformBlock(Func <TInput, Task <TOutput> > transform, ExecutionDataflowBlockOptions dataflowBlockOptions) :
this(null, transform, dataflowBlockOptions)
{
}
/// <summary>Initializes the <see cref="TransformManyBlock{TInput,TOutput}"/> with the specified function and <see cref="ExecutionDataflowBlockOptions"/>.</summary>
/// <param name="transformSync">The synchronous function to invoke with each data element received.</param>
/// <param name="transformAsync">The asynchronous function to invoke with each data element received.</param>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="TransformManyBlock{TInput,TOutput}"/>.</param>
/// <exception cref="System.ArgumentNullException">The <paramref name="transformSync"/> and <paramref name="transformAsync"/> are both null (Nothing in Visual Basic).</exception>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
private TransformManyBlock(Func <TInput, IEnumerable <TOutput> >?transformSync, Func <TInput, Task <IEnumerable <TOutput> > >?transformAsync, ExecutionDataflowBlockOptions dataflowBlockOptions)
{
// Validate arguments. It's ok for the filterFunction to be null, but not the other parameters.
if (transformSync == null && transformAsync == null)
{
throw new ArgumentNullException("transform");
}
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException(nameof(dataflowBlockOptions));
}
Debug.Assert(transformSync == null ^ transformAsync == null, "Exactly one of transformSync and transformAsync must be null.");
// Ensure we have options that can't be changed by the caller
dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone();
// Initialize onItemsRemoved delegate if necessary
Action <ISourceBlock <TOutput>, int>?onItemsRemoved = null;
if (dataflowBlockOptions.BoundedCapacity > 0)
{
onItemsRemoved = (owningSource, count) => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.ChangeBoundingCount(-count);
}
// Initialize source component
_source = new SourceCore <TOutput>(this, dataflowBlockOptions,
owningSource => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.Complete(exception: null, dropPendingMessages: true),
onItemsRemoved);
// If parallelism is employed, we will need to support reordering messages that complete out-of-order.
// However, a developer can override this with EnsureOrdered == false.
if (dataflowBlockOptions.SupportsParallelExecution && dataflowBlockOptions.EnsureOrdered)
{
_reorderingBuffer = new ReorderingBuffer <IEnumerable <TOutput> >(
this, (source, messages) => ((TransformManyBlock <TInput, TOutput>)source)._source.AddMessages(messages));
}
// Create the underlying target and source
if (transformSync != null) // sync
{
// If an enumerable function was provided, we can use synchronous completion, meaning
// that the target will consider a message fully processed as soon as the
// delegate returns.
_target = new TargetCore <TInput>(this,
messageWithId => ProcessMessage(transformSync, messageWithId),
_reorderingBuffer, dataflowBlockOptions, TargetCoreOptions.None);
}
else // async
{
Debug.Assert(transformAsync != null, "Incorrect delegate type.");
// If a task-based function was provided, we need to use asynchronous completion, meaning
// that the target won't consider a message completed until the task
// returned from that delegate has completed.
_target = new TargetCore <TInput>(this,
messageWithId => ProcessMessageWithTask(transformAsync, messageWithId),
_reorderingBuffer, dataflowBlockOptions, TargetCoreOptions.UsesAsyncCompletion);
}
// Link up the target half with the source half. In doing so,
// ensure exceptions are propagated, and let the source know no more messages will arrive.
// As the target has completed, and as the target synchronously pushes work
// through the reordering buffer when async processing completes,
// we know for certain that no more messages will need to be sent to the source.
_target.Completion.ContinueWith((completed, state) =>
{
var sourceCore = (SourceCore <TOutput>)state !;
if (completed.IsFaulted)
{
sourceCore.AddAndUnwrapAggregateException(completed.Exception !);
}
sourceCore.Complete();
}, _source, CancellationToken.None, Common.GetContinuationOptions(), TaskScheduler.Default);
// It is possible that the source half may fault on its own, e.g. due to a task scheduler exception.
// In those cases we need to fault the target half to drop its buffered messages and to release its
// reservations. This should not create an infinite loop, because all our implementations are designed
// to handle multiple completion requests and to carry over only one.
_source.Completion.ContinueWith((completed, state) =>
{
var thisBlock = ((TransformManyBlock <TInput, TOutput>)state !) as IDataflowBlock;
Debug.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion.");
thisBlock.Fault(completed.Exception !);
}, this, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default);
// Handle async cancellation requests by declining on the target
Common.WireCancellationToComplete(
dataflowBlockOptions.CancellationToken, Completion, state => ((TargetCore <TInput>)state !).Complete(exception: null, dropPendingMessages: true), _target);
#if FEATURE_TRACING
DataflowEtwProvider etwLog = DataflowEtwProvider.Log;
if (etwLog.IsEnabled())
{
etwLog.DataflowBlockCreated(this, dataflowBlockOptions);
}
#endif
}
public void MaxMessagesPerTaskTest()
{
var options = new ExecutionDataflowBlockOptions ();
foreach (var taskIds in GetTaskIdsForExecutionsOptions (options))
Assert.GreaterOrEqual (taskIds.Distinct ().Count (), 1);
options = new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1 };
foreach (var taskIds in GetTaskIdsForExecutionsOptions (options))
Assert.AreEqual (100, taskIds.Distinct ().Count ());
options = new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 2 };
foreach (var taskIds in GetTaskIdsForExecutionsOptions (options))
Assert.GreaterOrEqual (taskIds.Distinct ().Count (), taskIds.Length / 2);
options = new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 4 };
foreach (var taskIds in GetTaskIdsForExecutionsOptions (options))
Assert.GreaterOrEqual (taskIds.Distinct ().Count (), taskIds.Length / 4);
}
/// <summary> /// Create an adapter to change T into a dataflow message /// </summary> /// <param name="options">Execution options for the returned block</param> /// <returns>The adapter block</returns> public static IPropagatorBlock <T, DataflowMessage <T> > CreateAdapter(ExecutionDataflowBlockOptions options) => new TransformBlock <T, DataflowMessage <T> >(arg => _pool.Get().Init(arg), options);
/// <summary>Initializes the <see cref="ActionBlock{T}"/> with the specified <see cref="System.Action{T}"/> and <see cref="ExecutionDataflowBlockOptions"/>.</summary>
/// <param name="action">The action to invoke with each data element received.</param>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="ActionBlock{T}"/>.</param>
/// <exception cref="System.ArgumentNullException">The <paramref name="action"/> is null (Nothing in Visual Basic).</exception>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
public ActionBlock(Action <TInput> action, ExecutionDataflowBlockOptions dataflowBlockOptions) :
this((Delegate)action, dataflowBlockOptions)
{
}
private static bool TestQuickStop(bool testThrow)
{
bool passed = true;
CancellationTokenSource cts = new CancellationTokenSource();
var options = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token };
var propagator = new TransformBlock<ThrowOn, ThrowOn>(x => { Task.Delay(200).Wait(); return x; }, options);
var thrower = new ThrowerBlock();
// Post enough messages to require long processing
for (int i = 0; i < 2; i++)
propagator.Post(testThrow && i == 1 ? ThrowOn.OfferMessage : ThrowOn.TryReceive); // Throw on the second message
// Link the thrower
propagator.LinkTo(thrower);
// Once a message has been processed, cancel the propagator (if we are testing cancellation)
SpinWait.SpinUntil(() => thrower.LastOperation == ThrowOn.OfferMessage);
if (!testThrow) cts.Cancel();
// Wait for the propagator to complete
try
{
var ranToCompletion = propagator.Completion.Wait(10000);
passed = false;
Console.WriteLine("Task is faulted or canceled (finished: {0}) - FAILED", ranToCompletion ? "ran to copmpletion" : "still running");
}
catch (AggregateException ae)
{
passed = testThrow ? ae.InnerException is InvalidOperationException : ae.InnerException is TaskCanceledException;
ae.Handle(e => true);
Console.WriteLine("Task is faulted or canceled (exception) - {0}", passed ? "Passed" : "FAILED");
}
return passed;
}
/// <summary>
/// Create a logging thread to process the logging queue
/// </summary>
private void CreateLoggingEventQueue()
{
// We are creating a two-node dataflow graph here. The first node is a buffer, which will hold up to the number of
// logging events we have specified as the queueCapacity. The second node is the processor which will actually process each message.
// When the capacity of the buffer is reached, further attempts to send messages to it will block.
// The reason we can't just set the BoundedCapacity on the processing block is that ActionBlock has some weird behavior
// when the queue capacity is reached. Specifically, it will block new messages from being processed until it has
// entirely drained its input queue, as opposed to letting new ones in as old ones are processed. This is logged as
// a perf bug (305575) against Dataflow. If they choose to fix it, we can eliminate the buffer node from the graph.
var dataBlockOptions = new DataflowBlockOptions
{
BoundedCapacity = Convert.ToInt32(_queueCapacity)
};
_loggingQueue = new BufferBlock<object>(dataBlockOptions);
var executionDataBlockOptions = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1
};
_loggingQueueProcessor = new ActionBlock<object>(loggingEvent => LoggingEventProcessor(loggingEvent), executionDataBlockOptions);
var dataLinkOptions = new DataflowLinkOptions
{
PropagateCompletion = true
};
_loggingQueue.LinkTo(_loggingQueueProcessor, dataLinkOptions);
}
private void Initialize(
Action <KeyValuePair <TInput, long> > processMessageAction,
ExecutionDataflowBlockOptions dataflowBlockOptions,
[NotNull] ref SourceCore <TOutput>?source,
[NotNull] ref TargetCore <TInput>?target,
ref ReorderingBuffer <IEnumerable <TOutput> >?reorderingBuffer,
TargetCoreOptions targetCoreOptions)
{
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException(nameof(dataflowBlockOptions));
}
// Ensure we have options that can't be changed by the caller
dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone();
// Initialize onItemsRemoved delegate if necessary
Action <ISourceBlock <TOutput>, int>?onItemsRemoved = null;
if (dataflowBlockOptions.BoundedCapacity > 0)
{
onItemsRemoved = (owningSource, count) => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.ChangeBoundingCount(-count);
}
// Initialize source component
source = new SourceCore <TOutput>(this, dataflowBlockOptions,
owningSource => ((TransformManyBlock <TInput, TOutput>)owningSource)._target.Complete(exception: null, dropPendingMessages: true),
onItemsRemoved);
// If parallelism is employed, we will need to support reordering messages that complete out-of-order.
// However, a developer can override this with EnsureOrdered == false.
if (dataflowBlockOptions.SupportsParallelExecution && dataflowBlockOptions.EnsureOrdered)
{
reorderingBuffer = new ReorderingBuffer <IEnumerable <TOutput> >(
this, (source, messages) => ((TransformManyBlock <TInput, TOutput>)source)._source.AddMessages(messages));
}
// Create the underlying target and source
target = new TargetCore <TInput>(this, processMessageAction, _reorderingBuffer, dataflowBlockOptions, targetCoreOptions);
// Link up the target half with the source half. In doing so,
// ensure exceptions are propagated, and let the source know no more messages will arrive.
// As the target has completed, and as the target synchronously pushes work
// through the reordering buffer when async processing completes,
// we know for certain that no more messages will need to be sent to the source.
target.Completion.ContinueWith((completed, state) =>
{
var sourceCore = (SourceCore <TOutput>)state !;
if (completed.IsFaulted)
{
sourceCore.AddAndUnwrapAggregateException(completed.Exception !);
}
sourceCore.Complete();
}, source, CancellationToken.None, Common.GetContinuationOptions(), TaskScheduler.Default);
// It is possible that the source half may fault on its own, e.g. due to a task scheduler exception.
// In those cases we need to fault the target half to drop its buffered messages and to release its
// reservations. This should not create an infinite loop, because all our implementations are designed
// to handle multiple completion requests and to carry over only one.
source.Completion.ContinueWith((completed, state) =>
{
var thisBlock = ((TransformManyBlock <TInput, TOutput>)state !) as IDataflowBlock;
Debug.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion.");
thisBlock.Fault(completed.Exception !);
}, this, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default);
// Handle async cancellation requests by declining on the target
Common.WireCancellationToComplete(
dataflowBlockOptions.CancellationToken, Completion, state => ((TargetCore <TInput>)state !).Complete(exception: null, dropPendingMessages: true), target);
DataflowEtwProvider etwLog = DataflowEtwProvider.Log;
if (etwLog.IsEnabled())
{
etwLog.DataflowBlockCreated(this, dataflowBlockOptions);
}
}
/// <summary>Initializes the <see cref="TransformManyBlock{TInput,TOutput}"/> with the specified function and <see cref="ExecutionDataflowBlockOptions"/>.</summary>
/// <param name="transform">
/// The function to invoke with each data element received. All of the data from the returned in the <see cref="System.Collections.Generic.IEnumerable{TOutput}"/>
/// will be made available as output from this <see cref="TransformManyBlock{TInput,TOutput}"/>.
/// </param>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="TransformManyBlock{TInput,TOutput}"/>.</param>
/// <exception cref="System.ArgumentNullException">The <paramref name="transform"/> is null (Nothing in Visual Basic).</exception>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
public TransformManyBlock(Func <TInput, IEnumerable <TOutput> > transform, ExecutionDataflowBlockOptions dataflowBlockOptions) :
this(transform, null, dataflowBlockOptions)
{
}
