public State()
{
ChangeListener = DataflowBlock.NullTarget <State>();
needles = new Dictionary <NeedleKey, NeedleResult>();
sources = new Dictionary <string, IMagickImage>();
searchResults = new Dictionary <SearchKey, SearchResult>();
}
public async Task TestCancellationBeforeAndAfterCtor()
{
foreach (bool before in DataflowTestHelpers.BooleanValues)
{
var cts = new CancellationTokenSource();
if (before)
{
cts.Cancel();
}
var wob = new WriteOnceBlock <int>(null, new DataflowBlockOptions {
CancellationToken = cts.Token
});
if (!before)
{
cts.Cancel();
}
int ignoredValue;
IList <int> ignoredValues;
Assert.NotNull(wob.LinkTo(DataflowBlock.NullTarget <int>()));
Assert.False(wob.Post(42));
Task <bool> sendTask = wob.SendAsync(43);
Assert.True(sendTask.IsCompleted);
Assert.False(sendTask.Result);
Assert.False(wob.TryReceive(out ignoredValue));
Assert.False(((IReceivableSourceBlock <int>)wob).TryReceiveAll(out ignoredValues));
Assert.NotNull(wob.Completion);
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => wob.Completion);
}
}
// Token: 0x0600004F RID: 79 RVA: 0x0000350C File Offset: 0x0000170C
protected override async Task <ComplianceMessage> ProcessMessageInternal(ComplianceMessage message)
{
TransformBlock <ComplianceMessage, ComplianceMessage> transformer = null;
switch (message.ComplianceMessageType)
{
case ComplianceMessageType.RecordResult:
transformer = this.storeBlock.GetDataflowBlock(null);
goto IL_B1;
case ComplianceMessageType.RetrieveRequest:
transformer = this.retrievePayloadBlock.GetDataflowBlock(null);
goto IL_B1;
case ComplianceMessageType.EchoRequest:
transformer = this.echoBlock.GetDataflowBlock(null);
goto IL_B1;
}
throw new ArgumentException(string.Format("MessageType:{0} is not supported", message.ComplianceMessageType));
IL_B1:
await DataflowBlock.SendAsync <ComplianceMessage>(transformer, message);
ComplianceMessage response = await DataflowBlock.ReceiveAsync <ComplianceMessage>(transformer);
transformer.Complete();
return(response);
}
public async Task TestPrecancellation3()
{
var b = new JoinBlock <int, int, int>(new GroupingDataflowBlockOptions
{
CancellationToken = new CancellationToken(canceled: true),
MaxNumberOfGroups = 1
});
Assert.NotNull(b.LinkTo(DataflowBlock.NullTarget <Tuple <int, int, int> >()));
Assert.False(b.Target1.Post(42));
Assert.False(b.Target2.Post(43));
Assert.False(b.Target2.Post(44));
Task <bool> t1 = b.Target1.SendAsync(42);
Task <bool> t2 = b.Target2.SendAsync(43);
Task <bool> t3 = b.Target2.SendAsync(44);
Assert.True(t1.IsCompleted);
Assert.False(t1.Result);
Assert.True(t2.IsCompleted);
Assert.False(t2.Result);
Assert.True(t3.IsCompleted);
Assert.False(t3.Result);
Tuple <int, int, int> ignoredValue;
IList <Tuple <int, int, int> > ignoredValues;
Assert.False(b.TryReceive(out ignoredValue));
Assert.False(b.TryReceiveAll(out ignoredValues));
Assert.Equal(expected: 0, actual: b.OutputCount);
Assert.NotNull(b.Completion);
b.Complete();
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => b.Completion);
}
public async Task TestPrecanceled()
{
var bb = new BufferBlock <int>(
new DataflowBlockOptions {
CancellationToken = new CancellationToken(canceled: true)
});
int ignoredValue;
IList <int> ignoredValues;
IDisposable link = bb.LinkTo(DataflowBlock.NullTarget <int>());
Assert.NotNull(link);
link.Dispose();
Assert.False(bb.Post(42));
var t = bb.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
Assert.Equal(expected: 0, actual: bb.Count);
Assert.False(bb.TryReceiveAll(out ignoredValues));
Assert.False(bb.TryReceive(out ignoredValue));
Assert.NotNull(bb.Completion);
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => bb.Completion);
bb.Complete(); // just make sure it doesn't throw
}
public SourceLoader(string project, string hostUrl, Action <string> onFinishFile = null, ConcurrentBag <string> loadedFiles = null)
{
OnFinishFile = onFinishFile;
LoadedFiles = loadedFiles ?? LoadedFiles;
Project = project;
if (hostUrl != null)
{
if (hostUrl.EndsWith("/"))
{
HostUrl = hostUrl;
}
else
{
HostUrl = hostUrl + "/";
}
}
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
var blockOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = ExecutionDataflowBlockOptions.Unbounded, EnsureOrdered = false
};
LoadBlock = new TransformBlock <CodeFile, CodeFile>(OnLoadBlock, blockOptions);
ParseBlock = new TransformBlock <CodeFile, CodeFile>(OnParseBlock, blockOptions);
SaveBlock = new ActionBlock <CodeFile>(OnSaveBlock, blockOptions);
LoadBlock.LinkTo(ParseBlock, linkOptions, (cf) => cf != null);
LoadBlock.LinkTo(DataflowBlock.NullTarget <CodeFile>());
ParseBlock.LinkTo(SaveBlock, linkOptions);
}
/// <summary>
/// Creates the throttled transform many.
/// </summary>
/// <typeparam name="TIn">The type of the in.</typeparam>
/// <typeparam name="TOut">The type of the out.</typeparam>
/// <param name="transform">The transform.</param>
/// <param name="dataflowBlockOptions">The dataflow block options.</param>
/// <returns></returns>
public static IPropagatorBlock <TIn, TOut> CreateThrottledTransformMany <TIn, TOut>(
Func <TIn, IEnumerable <TOut> > transform,
ExecutionDataflowBlockOptions dataflowBlockOptions
)
{
var source = new BufferBlock <TOut>(dataflowBlockOptions);
var target = new ActionBlock <TIn>(@in => {
foreach (var @out in transform(@in))
{
source.SendAsync(@out).Wait();
}
});
target.Completion.ContinueWith(completion => {
if (completion.IsFaulted)
{
((IDataflowBlock)source).Fault(completion.Exception);
}
else
{
source.Complete();
}
});
return(DataflowBlock.Encapsulate(target, source));
}
public LoopDataflow3()
{
InputMessageBlock = new TransformBlock <Message, Message>(async msg => await InputMessage(msg));
HandleMessageBlock = new TransformBlock <Message, Message>(async msg => await HandleMessage(msg));
OutputMessageBlock = new ActionBlock <Message>(msg => OutputMessage(msg));
var linkOptions = new DataflowLinkOptions()
{
PropagateCompletion = false
};
InputMessageBlock.LinkTo(HandleMessageBlock, linkOptions);
HandleMessageBlock.LinkTo(OutputMessageBlock, linkOptions, msg => msg.WasProcessed == true);
HandleMessageBlock.LinkTo(DataflowBlock.NullTarget <Message>(), msg => {
throw new InvalidOperationException("Messages are being dropped.");
});
InputMessageBlock.Completion.ContinueWith(async tsk => {
await HandleMessageIsComplete();
HandleMessageBlock.Complete();
});
HandleMessageBlock.Completion.ContinueWith(tsk => {
OutputMessageBlock.Complete();
});
DebuggingLoop();
}
public static IPropagatorBlock <File, FileLine> GetFileLinesEnumeratorBlock()
{
var resultsBlock = new BufferBlock <FileLine>();
var actionBlock = new ActionBlock <File>(
async file =>
{
using (var reader = new System.IO.StreamReader(new System.IO.FileStream(
file.FullPath,
System.IO.FileMode.Open,
System.IO.FileAccess.Read,
System.IO.FileShare.Read,
bufferSize: 4096,
useAsync: true)))
{
string line;
var row = 1;
while ((line = await reader.ReadLineAsync()) != null)
{
if (!string.IsNullOrWhiteSpace(line))
{
resultsBlock.Post(new FileLine(file, row, line));
}
row++;
}
}
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism
});
actionBlock.PropagateCompleted(resultsBlock);
return(DataflowBlock.Encapsulate(actionBlock, resultsBlock));
}
public static IPropagatorBlock <T, T> CreateGuaranteedBroadcastBlock <T>(IEnumerable <ITargetBlock <T> > targets, bool propagateCompletion)
{
var source = new BufferBlock <T>();
var target = new ActionBlock <T>(async item =>
{
foreach (var targetBlock in targets)
{
await targetBlock.SendAsync(item);
}
}, new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 1
});
if (propagateCompletion)
{
target.Completion.ContinueWith(async delegate
{
foreach (var targetBlock in targets)
{
targetBlock.Complete();
await targetBlock.Completion;
}
});
}
return(DataflowBlock.Encapsulate(target, source));
}
public void NullTargetTest()
{
var target = DataflowBlock.NullTarget <int> ();
Assert.IsTrue(target.Post(1));
var source = new TestSourceBlock <int> ();
var header = new DataflowMessageHeader(1);
source.AddMessage(header, 2);
Assert.IsFalse(source.WasConsumed(header));
Assert.AreEqual(DataflowMessageStatus.Accepted,
target.OfferMessage(header, 2, source, true));
Assert.IsTrue(source.WasConsumed(header));
Assert.IsFalse(target.Completion.Wait(100));
target.Complete();
Assert.IsFalse(target.Completion.Wait(100));
target.Fault(new Exception());
Assert.IsFalse(target.Completion.Wait(100));
}
public async Task TestPrecancellation()
{
var b = new BatchBlock <int>(42, new GroupingDataflowBlockOptions
{
CancellationToken = new CancellationToken(canceled: true),
MaxNumberOfGroups = 1
});
Assert.Equal(expected: 42, actual: b.BatchSize);
Assert.NotNull(b.LinkTo(DataflowBlock.NullTarget <int[]>()));
Assert.False(b.Post(42));
Task <bool> t = b.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
int[] ignoredValue;
IList <int[]> ignoredValues;
Assert.False(b.TryReceive(out ignoredValue));
Assert.False(b.TryReceiveAll(out ignoredValues));
Assert.Equal(expected: 0, actual: b.OutputCount);
Assert.NotNull(b.Completion);
b.Complete(); // verify doesn't throw
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => b.Completion);
}
public static IPropagatorBlock <T, T[]> CreateBlock(uint batchSize)
{
var executionOptions = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1,
};
var destination = new BufferBlock <T[]>(executionOptions);
Queue <T> queue = new Queue <T>();
var queuer = new ActionBlock <T>(x =>
{
if (queue.Count >= batchSize)
{
var element = queue.ToArray();
queue.Clear();
destination.SendChecked(element);
}
queue.Enqueue(x);
}, new ExecutionDataflowBlockOptions {
BoundedCapacity = (int)batchSize, MaxDegreeOfParallelism = 1
});
//insertBat.LinkTo(transformerBlock, new DataflowLinkOptions { PropagateCompletion = true });
queuer.Completion.ContinueWith(x =>
{
var element = queue.ToArray();
queue.Clear();
destination.SendChecked(element);
destination.Complete();
});
var outputBlock = DataflowBlock.Encapsulate(queuer, destination);
return(outputBlock);
}
/// <summary>
/// Initializes a new instance of the <see cref="ResendMessagesChannelModule"/> class.
/// </summary>
/// <param name="resendMessageTryCount">The resend message try count.</param>
/// <param name="resendMessageInterval">The resend message interval.</param>
/// <param name="filterByDestination">if set to <c>true</c> [filter by destination].</param>
/// <exception cref="System.ArgumentOutOfRangeException"></exception>
public ResendMessagesChannelModule(int resendMessageTryCount, TimeSpan resendMessageInterval, bool filterByDestination = false)
{
if (resendMessageTryCount <= 0)
{
throw new ArgumentOutOfRangeException(nameof(resendMessageTryCount));
}
_resendMessageTryCount = resendMessageTryCount;
_resendMessageInterval = resendMessageInterval;
_filterByDestination = filterByDestination;
_sentMessageDictionary = new ConcurrentDictionary <MessageIdDestination, SentMessage>();
_inputBlock = new BufferBlock <SentMessage>();
_waitForRetryBlock = new TransformBlock <SentMessage, SentMessage>(
m => WaitForRetryAsync(m),
new ExecutionDataflowBlockOptions()
{
BoundedCapacity = DataflowBlockOptions.Unbounded
});
_resendBlock = new ActionBlock <SentMessage>(
ResendMessageAsync,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1
});
_discardBlock = DataflowBlock.NullTarget <SentMessage>();
_inputBlock.LinkTo(_waitForRetryBlock);
_waitForRetryBlock.LinkTo(_discardBlock, m => m == null);
_bindSemaphore = new SemaphoreSlim(1, 1);
}
private static IPropagatorBlock <long, long> Create3AverageNode(Func <long[], long> f)
{
const int BATCH_SIZE = 3;
// create
var target = new BatchBlock <long>(BATCH_SIZE);
var source = new TransformBlock <long[], long>(x => f(x)); // f = LongOperationArray
// link
target.LinkTo(source);
// completion
target.Completion.ContinueWith(completion =>
{
if (completion.IsFaulted)
{
((IDataflowBlock)source).Fault(completion.Exception);
}
else
{
source.Complete();
}
});
return(DataflowBlock.Encapsulate(target, source));
}
private TransformBlock <QueuedTransaction, QueuedTransaction> CreateQueuedTransactionBufferBlock()
{
var executionDataFlowBlockOptions = new ExecutionDataflowBlockOptions
{
BoundedCapacity = _transactionOptions.PoolLimit,
MaxDegreeOfParallelism = _transactionOptions.PoolParallelismDegree
};
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
var updateBucketIndexTransformBlock =
new TransformBlock <QueuedTransaction, QueuedTransaction>(UpdateBucketIndex,
executionDataFlowBlockOptions);
int i = 0;
while (i < _transactionOptions.PoolParallelismDegree)
{
var validationTransformBlock = new ActionBlock <QueuedTransaction>(
async queuedTransaction =>
await ProcessQueuedTransactionAsync(queuedTransaction, AcceptTransactionAsync),
new ExecutionDataflowBlockOptions
{
BoundedCapacity = _transactionOptions.PoolLimit,
EnsureOrdered = false
});
var index = i;
updateBucketIndexTransformBlock.LinkTo(validationTransformBlock, linkOptions,
queuedTransaction => queuedTransaction.BucketIndex == index);
i++;
}
updateBucketIndexTransformBlock.LinkTo(DataflowBlock.NullTarget <QueuedTransaction>());
return(updateBucketIndexTransformBlock);
}
public async Task TestPrecanceled()
{
var bb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable,
new ExecutionDataflowBlockOptions {
CancellationToken = new CancellationToken(canceled: true)
});
int ignoredValue;
IList <int> ignoredValues;
IDisposable link = bb.LinkTo(DataflowBlock.NullTarget <int>());
Assert.NotNull(link);
link.Dispose();
Assert.False(bb.Post(42));
var t = bb.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
Assert.False(bb.TryReceiveAll(out ignoredValues));
Assert.False(bb.TryReceive(out ignoredValue));
Assert.NotNull(bb.Completion);
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => bb.Completion);
bb.Complete(); // just make sure it doesn't throw
}
/// <summary>
/// Linking source and target node properties based on their types.
/// </summary>
/// <param name="nodes">Nodes to inspect.</param>
public static void LinkNodes(IEnumerable <INode> nodes)
{
ForEachProperty(typeof(ISourceBlock <>), nodes, (sourceBlockType, sourceBlock) =>
{
var targetBlocks = new List <dynamic>();
ForEachProperty(typeof(ITargetBlock <>), nodes, (targetBlockType, targetBlock) =>
{
if (sourceBlockType.GenericTypeArguments[0].IsAssignableTo(targetBlockType.GenericTypeArguments[0]))
{
targetBlocks.Add(targetBlock);
}
});
if (targetBlocks.Count > 1)
{
var broadcastBlockType = typeof(BroadcastBlock <>).MakeGenericType(sourceBlockType.GenericTypeArguments);
// CreateInstance can return null only for Nullable types
dynamic broadcastBlock = Activator.CreateInstance(broadcastBlockType, new object?[] { null }) !;
DataflowBlock.LinkTo(sourceBlock, broadcastBlock);
sourceBlock = broadcastBlock;
}
foreach (dynamic targetBlock in targetBlocks)
{
DataflowBlock.LinkTo(sourceBlock, targetBlock);
}
});
}
public static IPropagatorBlock <T, T> CreateFilteringBlock <T>()
where T : IComparable <T>, new()
{
T maxElement = default;
var source = new BufferBlock <T>();
var target = new ActionBlock <T>(async item =>
{
if (item.CompareTo(maxElement) > 0)
{
await source.SendAsync(item);
maxElement = item;
}
});
target.Completion.ContinueWith(a =>
{
if (a.IsFaulted)
{
((ITargetBlock <T>)source).Fault(a.Exception);
}
else
{
source.Complete();
}
});
return(DataflowBlock.Encapsulate(target, source));
}
public static IPropagatorBlock <TIn, TIn> CreateFilterBlock <TIn>(Predicate <TIn> predicate)
{
var outputBuffer = new BufferBlock <TIn>();
var filterAction = new ActionBlock <TIn>(
async(input) =>
{
if (predicate(input))
{
await outputBuffer
.SendAsync(input)
.ConfigureAwait(false);
}
});
// Link faulted status or trigger completion at time of filter ActionBlock achieving completion.
filterAction.Completion.ContinueWith(
task =>
{
if (task.IsFaulted)
{
((ITargetBlock <TIn>)outputBuffer).Fault(task.Exception?.Flatten().InnerException);
}
else
{
outputBuffer.Complete();
}
});
return(DataflowBlock.Encapsulate(filterAction, outputBuffer));
}
/// <summary>
/// Create non-sequential transform behavior by composition of blocks.
/// </summary>
/// <typeparam name="Tin">The type of the in.</typeparam>
/// <typeparam name="Tout">The type of the out.</typeparam>
/// <param name="act">The act.</param>
/// <param name="options">The options.</param>
/// <returns></returns>
private static IPropagatorBlock <Tin, Tout> CreateMultiTransform <Tin, Tout>(
Func <Tin, Task <Tout> > act,
ExecutionDataflowBlockOptions options)
{
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
var input = new BufferBlock <Tin>(); // get the input
var output = new BufferBlock <Tout>(); // keep the output
int degreeOfParallelism = options.MaxDegreeOfParallelism;
options.MaxDegreeOfParallelism = 1; // reset to single task
options.BoundedCapacity = 1; // otherwise the first one will take it all
// instead of single block with multi-task
// having multi-blocks with single task
// each block map the input to the output
var blockes = new Task[degreeOfParallelism]; // completion of all blocks
for (int i = 0; i < degreeOfParallelism; i++)
{
var block = new TransformBlock <Tin, Tout>(act, options);
blockes[i] = block.Completion;
input.LinkTo(block, linkOptions);
block.LinkTo(output);
}
Task _ = Task.WhenAll(blockes).ContinueWith(m => output.Complete());
return(DataflowBlock.Encapsulate(input, output)); // wrap the functionality
}
/// <summary>
/// Handles the completion of all blocks up to this one.
/// </summary>
/// <param name="obj"></param>
private async Task HandleCompletion(Task obj)
{
//Link to null, so that the flow completes
if (_lastTransformerBlockOnCompletion != null)
{
var nullt = DataflowBlock.NullTarget <TOut>();
_lastTransformerBlockOnCompletion.LinkTo(nullt);
}
if (_transformerOnCompletion != null)
{
IEnumerable <TOut> elements = GetCollectedItems();
if (elements != null)
{
//Post all items to the first transformer
foreach (TOut element in elements)
{
if (element == null)
{
continue;
}
_transformerOnCompletion.SendAsync(element).Wait();
}
}
else
{
}
//Set it to complete
_transformerOnCompletion.Complete();
//Wait for the last transformer to complete
_lastTransformerBlockOnCompletion.Completion.Wait();
}
await WaitForContinuingTasks();
}
public TPLPipelineWithAwaitAttempt2 <TIn, TOut> AddStep <TLocalIn, TLocalOut>(Func <TLocalIn, TLocalOut> stepFunc)
{
var step = new TransformBlock <TC <TLocalIn, TOut>, TC <TLocalOut, TOut> >((tc) =>
{
try
{
return(new TC <TLocalOut, TOut>(stepFunc(tc.Input), tc.TaskCompletionSource));
}
catch (Exception e)
{
tc.TaskCompletionSource.SetException(e);
return(new TC <TLocalOut, TOut>(default(TLocalOut), tc.TaskCompletionSource));
}
});
if (_transformBlocks.Count > 0)
{
var lastStep = _transformBlocks.Last();
var targetBlock = (lastStep as ISourceBlock <TC <TLocalIn, TOut> >);
targetBlock.LinkTo(step, new DataflowLinkOptions(),
tc => !tc.TaskCompletionSource.Task.IsFaulted);
targetBlock.LinkTo(DataflowBlock.NullTarget <TC <TLocalIn, TOut> >(), new DataflowLinkOptions(),
tc => tc.TaskCompletionSource.Task.IsFaulted);
}
_transformBlocks.Add(step);
return(this);
}
/// <summary>
/// Links an action to this block. Usefull if it's an action block.
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="target"></param>
/// <returns></returns>
public IFlowBlock BroadcastTo(ITargetBlock <TOut> target, bool linkToCompletion = true)
{
ItemProcessed += (x) =>
{
DataflowBlock.SendAsync(target, x).Wait();
};
if (linkToCompletion)
{
AddCompletionTask(target.Completion);
}
var actionBlock = this.GetProcessingBlock();
actionBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted)
{
target.Fault(t.Exception);
}
else
{
target.Complete();
}
});
//_lastLinkedBlock = target;
return(this);
}
private IPropagatorBlock <int, Data> UseLinesReaderAndParser(StreamReader peopleJsonStream, DataPool dataPool, CancellationToken cancellation)
{
// Create blocks
// NOTE:
// - extract part which must be single-thread
// - ability to replace just reading (e.g. from db)
var readLinesBlock = DataflowFacade.TransformManyBlock <int, Data>(
"ReadLines",
x => - 1,
x => _streamLinesReader.Read(peopleJsonStream, x)
.Select(line =>
{
var data = dataPool.Rent();
data.PersonJson = line;
return(data);
}),
cancellation);
// NOTE: can be multi-thread
var parseDataBlock = DataflowFacade.TransformBlock(
"ParseData",
Data.IdGetter,
x => _dataParser.Parse(x),
cancellation);
// Link blocks
readLinesBlock.LinkWithCompletion(parseDataBlock);
return(DataflowBlock.Encapsulate(readLinesBlock, parseDataBlock));
}
/// <summary>
/// Create a feature generator block, with all the current feature generators.
/// </summary>
/// <param name="threadCount"></param>
/// <returns></returns>
public IPropagatorBlock <TIn, IEnumerable <KeyValuePair <string, object> > > CreateFeaturePairsBlock()
{
//Dataflow: poster -> each transformer -> buffer
var buffer = new BufferBlock <IEnumerable <KeyValuePair <string, object> > >();
// The target part receives data and adds them to the queue.
var transformers = _featureGenerators
.Select(x =>
{
var transformer =
new TransformBlock <TIn, IEnumerable <KeyValuePair <string, object> > >(x);
transformer.LinkTo(buffer);
return(transformer);
});
var postOptions = new ExecutionDataflowBlockOptions();
postOptions.MaxDegreeOfParallelism = _threadCount;
//Post an item to each transformer
var poster = new ActionBlock <TIn>(doc =>
{
foreach (var transformer in transformers)
{
transformer.Post(doc);
}
}, postOptions);
// Return a IPropagatorBlock<T, T[]> object that encapsulates the
// target and source blocks.
return(DataflowBlock.Encapsulate(poster, buffer));
}
private static IPropagatorBlock <BulkOperation, byte[]> CreateBulkRequestBodyBlock(int targetSizeInBytes, bool omitTypeHeaders)
{
var transformOpts = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1
};
var stream = new MemoryStream();
var writer = new BulkWriter(stream, omitTypeHeaders);
var source = new BufferBlock <byte[]>();
var target = new ActionBlock <BulkOperation>(async op =>
{
op.Write(writer);
writer.Flush();
if (stream.Length >= targetSizeInBytes)
{
stream.Position = 0;
await source.SendAsync(stream.ToArray()).ConfigureAwait(false);
stream.SetLength(0);
writer = new BulkWriter(stream, omitTypeHeaders);
}
}, transformOpts);
target.Completion.ContinueWith(async t =>
{
if (stream.Length > 0)
{
stream.Position = 0;
await source.SendAsync(stream.ToArray()).ConfigureAwait(false);
stream.SetLength(0);
}
source.Complete();
});
return(DataflowBlock.Encapsulate(target, source));
}
private static IPropagatorBlock <long, long> Create2CrossNode_2(Func <long[], long> f)
{
const int BATCHSIZE = 2;
// creation
var target = new BatchBlock <long>(BATCHSIZE, new GroupingDataflowBlockOptions()
{
Greedy = true
});
var source = new TransformBlock <long[], long>(x => f(x)); // f = LongOperationArray
// linking
target.LinkTo(source);
// completion
target.Completion.ContinueWith(completion =>
{
if (completion.IsFaulted)
{
((IDataflowBlock)source).Fault(completion.Exception);
}
else
{
source.Complete();
}
});
return(DataflowBlock.Encapsulate(target, source));
}
private static ITargetBlock <T> CreateChooseTarget <T>()
{
var slt = new StoreLinkedTarget <T>();
DataflowBlock.Choose(slt, i => { }, new BufferBlock <T>(), i => { });
return(slt.GetLinkedTarget());
}
public static IMessagePipeline <IMessage <T>, IMessageEnumerable <T> > Batch <T>(
this IMessagePipeline <IMessage <T>, IMessage <T> > pipeline,
int size,
TimeSpan time)
{
if (size <= 1)
{
throw new ArgumentException("Buffer size must be greater that 1");
}
var batch = new BatchBlock <IMessage <T> >(size, new GroupingDataflowBlockOptions
{
EnsureOrdered = true,
BoundedCapacity = size
});
var timer = time.TotalMilliseconds > 0
? new Timer(_ => batch.TriggerBatch(), null, (int)time.TotalMilliseconds, Timeout.Infinite)
: null;
var transform = new TransformBlock <IMessage <T>[], IMessageEnumerable <T> >(x =>
{
timer?.Change((int)time.TotalMilliseconds, Timeout.Infinite);
return(new KafkaMessageEnumerable <T>(x));
},
new ExecutionDataflowBlockOptions
{
EnsureOrdered = true,
BoundedCapacity = 1
});
batch.LinkTo(transform);
return(pipeline.Block(DataflowBlock.Encapsulate(batch, transform)));
}
