public void ProduceLogs(int count, int buffSize)
{
var bufferOptions = new DataflowBlockOptions() { BoundedCapacity = buffSize };
var writerOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 10, MaxDegreeOfParallelism = 1, MaxMessagesPerTask = 10, SingleProducerConstrained = true };
LogGenerator g = new LogGenerator();
var file = new StreamWriter("basic.async.buff.log", false);
BufferBlock<string> buffer = new BufferBlock<string>(bufferOptions);
ActionBlock<string> writer = new ActionBlock<string>(s => file.WriteLine(s), writerOptions);
buffer.LinkTo(writer, new DataflowLinkOptions() { PropagateCompletion = true });
for (int i = 0; i < count; i++)
{
g.Next();
var line = string.Format(g.FormatStr, g.Param1, g.Param2, g.Param3, g.Param4, g.Param5, g.Param6);
writer.SendAsync(line).Wait();
}
buffer.Complete();
Completed = writer.Completion.ContinueWith(t => file.Close());
}
protected OutgoingQueueBase(
CompletionHelper compHelper, Func <bool> externalCompleteTester,
Action <int> decreaseItemsCount, DataflowBlockOptions options)
{
IsProcessing = new AtomicBoolean();
Store = new ConcurrentQueue <T> ();
Outgoing = new BlockingCollection <T> (Store);
this.compHelper = compHelper;
this.externalCompleteTester = externalCompleteTester;
this.options = options;
this.decreaseItemsCount = decreaseItemsCount;
}
public BufferBlock(DataflowBlockOptions dataflowBlockOptions)
{
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException("dataflowBlockOptions");
}
this.dataflowBlockOptions = dataflowBlockOptions;
this.messageBox = new PassingMessageBox <T> (messageQueue, compHelper, () => outgoing.IsCompleted, ProcessQueue, dataflowBlockOptions);
this.outgoing = new MessageOutgoingQueue <T> (compHelper, () => messageQueue.IsCompleted);
this.vault = new MessageVault <T> ();
}
protected MessageBox(
ITargetBlock <TInput> target, BlockingCollection <TInput> messageQueue,
CompletionHelper compHelper, Func <bool> externalCompleteTester,
DataflowBlockOptions options, bool greedy = true, Func <bool> canAccept = null)
{
this.Target = target;
this.CompHelper = compHelper;
this.MessageQueue = messageQueue;
this.externalCompleteTester = externalCompleteTester;
this.options = options;
this.greedy = greedy;
this.canAccept = canAccept;
}
public BatchBlock(int batchSize, DataflowBlockOptions dataflowBlockOptions)
{
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException("dataflowBlockOptions");
}
this.batchSize = batchSize;
this.dataflowBlockOptions = dataflowBlockOptions;
this.messageBox = new PassingMessageBox <T> (messageQueue, compHelper, () => outgoing.IsCompleted, BatchProcess, dataflowBlockOptions);
this.outgoing = new MessageOutgoingQueue <T[]> (compHelper, () => messageQueue.IsCompleted);
this.vault = new MessageVault <T[]> ();
}
/// <summary>
/// 初期化処理。
/// </summary>
/// <param name="settingObject"></param>
/// <param name="token"></param>
public void Init(dynamic settingObject, CancellationToken token)
{
logger.Trace("Init Start");
var opt = new DataflowBlockOptions
{
CancellationToken = token,
};
var buffer = new BufferBlock<PastaLog>(opt);
var bloadcast = new BroadcastBlock<PastaLog>(CloneLog, opt);
buffer.LinkTo(bloadcast);
Target = buffer;
Source = bloadcast;
logger.Trace("Init End");
}
public BufferBlock(DataflowBlockOptions dataflowBlockOptions)
{
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException("dataflowBlockOptions");
}
this.dataflowBlockOptions = dataflowBlockOptions;
this.compHelper = new CompletionHelper(dataflowBlockOptions);
this.messageBox = new PassingMessageBox <T> (this, messageQueue, compHelper,
() => outgoing.IsCompleted, _ => ProcessQueue(), dataflowBlockOptions);
this.outgoing = new OutgoingQueue <T> (this, compHelper,
() => messageQueue.IsCompleted, messageBox.DecreaseCount,
dataflowBlockOptions);
}
/// <summary>Initializes the <see cref="WriteOnceBlock{T}"/> with the specified <see cref="DataflowBlockOptions"/>.</summary>
/// <param name="cloningFunction">
/// The function to use to clone the data when offered to other blocks.
/// This may be null to indicate that no cloning need be performed.
/// </param>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="WriteOnceBlock{T}"/>.</param>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
public WriteOnceBlock(Func <T, T> cloningFunction, DataflowBlockOptions dataflowBlockOptions)
{
// Validate arguments
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException("dataflowBlockOptions");
}
Contract.EndContractBlock();
// Store the option
_cloningFunction = cloningFunction;
_dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone();
// The target registry also serves as our ValueLock,
// and thus must always be initialized, even if the block is pre-canceled, as
// subsequent usage of the block may run through code paths that try to take this lock.
_targetRegistry = new TargetRegistry <T>(this);
// If a cancelable CancellationToken has been passed in,
// we need to initialize the completion task's TCS now.
if (dataflowBlockOptions.CancellationToken.CanBeCanceled)
{
_lazyCompletionTaskSource = new TaskCompletionSource <VoidResult>();
// If we've already had cancellation requested, do as little work as we have to
// in order to be done.
if (dataflowBlockOptions.CancellationToken.IsCancellationRequested)
{
_completionReserved = _decliningPermanently = true;
// Cancel the completion task's TCS
_lazyCompletionTaskSource.SetCanceled();
}
else
{
// Handle async cancellation requests by declining on the target
Common.WireCancellationToComplete(
dataflowBlockOptions.CancellationToken, _lazyCompletionTaskSource.Task, state => ((WriteOnceBlock <T>)state).Complete(), this);
}
}
#if FEATURE_TRACING
DataflowEtwProvider etwLog = DataflowEtwProvider.Log;
if (etwLog.IsEnabled())
{
etwLog.DataflowBlockCreated(this, dataflowBlockOptions);
}
#endif
}
/// <summary>Initializes the <see cref="BufferBlock{T}"/> with the specified <see cref="DataflowBlockOptions"/>.</summary>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="BufferBlock{T}"/>.</param>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
public BufferBlock(DataflowBlockOptions dataflowBlockOptions)
{
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException(nameof(dataflowBlockOptions));
}
Contract.EndContractBlock();
// Ensure we have options that can't be changed by the caller
dataflowBlockOptions = dataflowBlockOptions.DefaultOrClone();
// Initialize bounding state if necessary
Action <ISourceBlock <T>, int> onItemsRemoved = null;
if (dataflowBlockOptions.BoundedCapacity > 0)
{
onItemsRemoved = (owningSource, count) => ((BufferBlock <T>)owningSource).OnItemsRemoved(count);
_boundingState = new BoundingStateWithPostponedAndTask <T>(dataflowBlockOptions.BoundedCapacity);
}
// Initialize the source state
_source = new SourceCore <T>(this, dataflowBlockOptions,
owningSource => ((BufferBlock <T>)owningSource).Complete(),
onItemsRemoved);
// It is possible that the source half may fault on its own, e.g. due to a task scheduler exception.
// In those cases we need to fault the target half to drop its buffered messages and to release its
// reservations. This should not create an infinite loop, because all our implementations are designed
// to handle multiple completion requests and to carry over only one.
_source.Completion.ContinueWith((completed, state) =>
{
var thisBlock = ((BufferBlock <T>)state) as IDataflowBlock;
Debug.Assert(completed.IsFaulted, "The source must be faulted in order to trigger a target completion.");
thisBlock.Fault(completed.Exception);
}, this, CancellationToken.None, Common.GetContinuationOptions() | TaskContinuationOptions.OnlyOnFaulted, TaskScheduler.Default);
// Handle async cancellation requests by declining on the target
Common.WireCancellationToComplete(
dataflowBlockOptions.CancellationToken, _source.Completion, owningSource => ((BufferBlock <T>)owningSource).Complete(), this);
#if FEATURE_TRACING
DataflowEtwProvider etwLog = DataflowEtwProvider.Log;
if (etwLog.IsEnabled())
{
etwLog.DataflowBlockCreated(this, dataflowBlockOptions);
}
#endif
}
public BroadcastBlock(Func <T, T> cloningFunction,
DataflowBlockOptions dataflowBlockOptions)
{
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException("dataflowBlockOptions");
}
this.cloningFunction = cloningFunction;
this.dataflowBlockOptions = dataflowBlockOptions;
this.compHelper = new CompletionHelper(dataflowBlockOptions);
this.messageBox = new PassingMessageBox <T> (this, messageQueue, compHelper,
() => outgoing.IsCompleted, _ => BroadcastProcess(), dataflowBlockOptions);
this.outgoing = new BroadcastOutgoingQueue <T> (this, compHelper,
() => messageQueue.IsCompleted, messageBox.DecreaseCount,
dataflowBlockOptions, cloningFunction != null);
}
public WriteOnceBlock(Func <T, T> cloningFunction,
DataflowBlockOptions dataflowBlockOptions)
{
if (dataflowBlockOptions == null)
{
throw new ArgumentNullException("dataflowBlockOptions");
}
this.cloningFunction = cloningFunction;
this.dataflowBlockOptions = dataflowBlockOptions;
this.compHelper = CompletionHelper.GetNew(dataflowBlockOptions);
this.messageBox = new PassingMessageBox <T> (this, messageQueue, compHelper,
() => true, _ => BroadcastProcess(), dataflowBlockOptions,
canAccept: () => written.TrySet());
this.outgoing = new BroadcastOutgoingQueue <T> (this, compHelper,
() => messageQueue.IsCompleted, messageBox.DecreaseCount,
dataflowBlockOptions, cloningFunction != null);
}
/// <summary>
/// 初期化処理。
/// </summary>
/// <param name="settingObject"></param>
/// <param name="token"></param>
/// <returns></returns>
public void Init(dynamic settingObject, CancellationToken token)
{
logger.Trace("Init Start");
token.Register(Dispose);
var opt = new DataflowBlockOptions
{
CancellationToken = token,
};
var buffer = new BufferBlock<PastaLog>(opt);
var actOpt = new ExecutionDataflowBlockOptions()
{
CancellationToken = token,
SingleProducerConstrained = true,
};
var act = new ActionBlock<PastaLog>((a) => Save(a), actOpt);
buffer.LinkTo(act);
Target = buffer;
logger.Trace("Init End");
}
/// <summary>
/// Creates a new instance of ChannelBase
/// </summary>
/// <param name="transport"></param>
/// <param name="sendTimeout"></param>
/// <param name="buffersLimit"></param>
/// <param name="fillEnvelopeRecipients">Indicates if the from and to properties of sent and received envelopes should be filled with the session information if not defined.</param>
/// <param name="autoReplyPings">Indicates if the channel should reply automatically to ping request commands. In this case, the ping command are not returned by the ReceiveCommandAsync method.</param>
public ChannelBase(ITransport transport, TimeSpan sendTimeout, int buffersLimit, bool fillEnvelopeRecipients, bool autoReplyPings)
{
if (transport == null)
{
throw new ArgumentNullException("transport");
}
this.Transport = transport;
this.Transport.Closing += Transport_Closing;
_sendTimeout = sendTimeout;
_fillEnvelopeRecipients = fillEnvelopeRecipients;
_autoReplyPings = autoReplyPings;
_channelCancellationTokenSource = new CancellationTokenSource();
this.State = SessionState.New;
#if MONO
_messageBuffer = new AsyncQueue <Message> (buffersLimit, buffersLimit);
_commandBuffer = new AsyncQueue <Command> (buffersLimit, buffersLimit);
_notificationBuffer = new AsyncQueue <Notification> (buffersLimit, buffersLimit);
_sessionBuffer = new AsyncQueue <Session> (1, 1);
#else
var bufferOptions = new System.Threading.Tasks.Dataflow.DataflowBlockOptions()
{
BoundedCapacity = buffersLimit
};
_messageBuffer = new BufferBlock <Message>(bufferOptions);
_commandBuffer = new BufferBlock <Command>(bufferOptions);
_notificationBuffer = new BufferBlock <Notification>(bufferOptions);
_sessionBuffer = new BufferBlock <Session>(
new DataflowBlockOptions()
{
BoundedCapacity = 1
});
#endif
}
public static Task <int> Choose <T1, T2> (ISourceBlock <T1> source1,
Action <T1> action1,
ISourceBlock <T2> source2,
Action <T2> action2,
DataflowBlockOptions dataflowBlockOptions)
{
if (source1 == null)
{
throw new ArgumentNullException("source1");
}
if (source2 == null)
{
throw new ArgumentNullException("source2");
}
var chooser = new ChooserBlock <T1, T2, object> (action1, action2, null, dataflowBlockOptions);
source1.LinkTo(chooser.Target1);
source2.LinkTo(chooser.Target2);
return(chooser.Completion);
}
/// <summary>Initializes the <see cref="BufferBlock{T}"/> with the specified <see cref="DataflowBlockOptions"/>.</summary>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="BufferBlock{T}"/>.</param>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
public BufferBlock(DataflowBlockOptions dataflowBlockOptions !!)
{
public void RunWriteOnceBlockConformanceTests()
{
bool passed = true, localPassed = true;
{
// Test posting then receiving
localPassed = true;
var wob = new WriteOnceBlock<int>(i => i);
int successfulPosts = 0;
for (int i = 10; i <= 20; i++)
{
successfulPosts += wob.Post(i) ? 1 : 0;
}
localPassed |= successfulPosts == 1;
localPassed |= wob.Receive() == 10;
Console.WriteLine("{0}: Posting then receiving", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
{
// Test receiving then posting
localPassed = true;
var wob = new WriteOnceBlock<int>(i => i);
Task.Factory.StartNew(() =>
{
Task.Delay(1000).Wait();
wob.Post(42);
});
localPassed |= wob.Receive() == 42;
localPassed |= wob.Post(43) == false;
wob.Completion.Wait();
Console.WriteLine("{0}: Receiving then posting", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
{
// Test broadcasting
localPassed = true;
var wob = new WriteOnceBlock<int>(i => i + 1);
var tb1 = new TransformBlock<int, int>(i => i);
var tb2 = new TransformBlock<int, int>(i => i);
var tb3 = new TransformBlock<int, int>(i => i);
wob.LinkTo(tb1);
wob.LinkTo(tb2);
wob.LinkTo(tb3);
wob.Post(42);
localPassed |= tb1.Receive() == 43;
localPassed |= tb2.Receive() == 43;
localPassed |= tb3.Receive() == 43;
Console.WriteLine("{0}: Broadcasting", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
{
// Test using a precanceled token
localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new DataflowBlockOptions { CancellationToken = cts.Token };
var wob = new WriteOnceBlock<int>(i => i, dbo);
int ignoredValue;
IList<int> ignoredValues;
localPassed &= wob.LinkTo(new ActionBlock<int>(delegate { })) != null;
localPassed &= wob.SendAsync(42).Result == false;
localPassed &= ((IReceivableSourceBlock<int>)wob).TryReceiveAll(out ignoredValues) == false;
localPassed &= wob.Post(42) == false;
localPassed &= wob.TryReceive(out ignoredValue) == false;
localPassed &= wob.Completion != null;
wob.Complete();
}
catch (Exception)
{
localPassed = false;
}
Console.WriteLine("{0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
{
// Test using token canceled after construction
localPassed = true;
try
{
var cts = new CancellationTokenSource();
var dbo = new DataflowBlockOptions { CancellationToken = cts.Token };
var wob = new WriteOnceBlock<int>(i => i, dbo);
cts.Cancel();
int ignoredValue;
IList<int> ignoredValues;
localPassed &= wob.LinkTo(new ActionBlock<int>(delegate { })) != null;
localPassed &= wob.SendAsync(42).Result == false;
localPassed &= ((IReceivableSourceBlock<int>)wob).TryReceiveAll(out ignoredValues) == false;
localPassed &= wob.Post(42) == false;
localPassed &= wob.TryReceive(out ignoredValue) == false;
localPassed &= wob.Completion != null;
wob.Complete();
}
catch (Exception)
{
localPassed = false;
}
Console.WriteLine("{0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
Assert.True(passed, "Test failed.");
}
public void TestCanceledLinking()
{
bool passed = true;
var cts = new CancellationTokenSource();
cts.Cancel();
var options = new DataflowBlockOptions { CancellationToken = cts.Token };
var writeOnce = new WriteOnceBlock<int>(x => x, options);
var target = new ActionBlock<int>(x => { });
try
{
writeOnce.LinkTo(target);
Console.WriteLine("Completed without exception - Passed");
}
catch (Exception)
{
passed = false;
Console.WriteLine("Completed without exception - FAILED");
}
Assert.True(passed, "Test failed.");
}
//[Fact(Skip = "Outerloop")]
public void RunBroadcastBlockConformanceTests()
{
bool localPassed = true;
{
// Test posting then receiving
localPassed = true;
var bb = new BroadcastBlock<int>(i => i);
for (int i = 0; i < 2; i++) bb.Post(i);
Task.Delay(1).Wait();
localPassed |= bb.Receive() == 1;
Assert.True(localPassed, string.Format("{0}: Posting then receiving", localPassed ? "Success" : "Failure"));
}
{
// Test receiving then posting
localPassed = true;
var bb = new BroadcastBlock<int>(i => i);
Task.Factory.StartNew(() =>
{
Task.Delay(1).Wait();
bb.Post(42);
});
localPassed |= bb.Receive() == 42;
Assert.True(localPassed, string.Format("{0}: Receiving then posting", localPassed ? "Success" : "Failure"));
}
{
// Test broadcasting
localPassed = true;
var bb = new BroadcastBlock<int>(i => i + 1);
var tb1 = new TransformBlock<int, int>(i => i);
var tb2 = new TransformBlock<int, int>(i => i);
var tb3 = new TransformBlock<int, int>(i => i);
bb.LinkTo(tb1);
bb.LinkTo(tb2);
bb.LinkTo(tb3);
for (int i = 0; i < 2; i++)
{
bb.Post(i);
}
for (int i = 0; i < 2; i++)
{
localPassed |= tb1.Receive() == i + 1;
localPassed |= tb2.Receive() == i + 1;
localPassed |= tb3.Receive() == i + 1;
}
Assert.True(localPassed, string.Format("{0}: Broadcasting", localPassed ? "Success" : "Failure"));
}
// Test using a precanceled token
{
localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new DataflowBlockOptions { CancellationToken = cts.Token };
var bb = new BroadcastBlock<int>(i => i, dbo);
int ignoredValue;
IList<int> ignoredValues;
localPassed &= bb.LinkTo(new ActionBlock<int>(delegate { })) != null;
localPassed &= bb.SendAsync(42).Result == false;
localPassed &= ((IReceivableSourceBlock<int>)bb).TryReceiveAll(out ignoredValues) == false;
localPassed &= bb.Post(42) == false;
localPassed &= bb.TryReceive(out ignoredValue) == false;
localPassed &= bb.Completion != null;
bb.Complete();
}
catch (Exception)
{
localPassed = false;
}
Assert.True(localPassed, string.Format("{0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"));
}
}
/// <summary>
/// Builds the internal buffers using the provided options.
/// </summary>
/// <param name="options"></param>
public PriorityBufferBlock(DataflowBlockOptions options) : this(options, options, options)
{
}
/// <summary>
/// Each internal BufferBlock is built using the provided options specified for that priority.
/// </summary>
/// <param name="highPriorityOptions"></param>
/// <param name="mediumPriorityOptions"></param>
/// <param name="lowPriorityOptions"></param>
public PriorityBufferBlock(DataflowBlockOptions highPriorityOptions, DataflowBlockOptions mediumPriorityOptions, DataflowBlockOptions lowPriorityOptions)
{
_highPriorityBuffer = new BufferBlock <T>(highPriorityOptions);
_mediumPriorityBuffer = new BufferBlock <T>(mediumPriorityOptions);
_lowPriorityBuffer = new BufferBlock <T>(lowPriorityOptions);
}
public static CompletionHelper GetNew(DataflowBlockOptions options)
{
return(new CompletionHelper(options));
}
/// <summary>Initializes the <see cref="WriteOnceBlock{T}"/> with the specified <see cref="DataflowBlockOptions"/>.</summary>
/// <param name="cloningFunction">
/// The function to use to clone the data when offered to other blocks.
/// This may be null to indicate that no cloning need be performed.
/// </param>
/// <param name="dataflowBlockOptions">The options with which to configure this <see cref="WriteOnceBlock{T}"/>.</param>
/// <exception cref="System.ArgumentNullException">The <paramref name="dataflowBlockOptions"/> is null (Nothing in Visual Basic).</exception>
public WriteOnceBlock(Func <T, T>?cloningFunction, DataflowBlockOptions dataflowBlockOptions !!)
{
public void RunBufferBlockConformanceTests()
{
bool localPassed;
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> bufferBlockFactory =
options =>
{
BufferBlock<int> bufferBlock = new BufferBlock<int>(options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
bufferBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = bufferBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token };
Assert.True(FeedTarget(bufferBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true));
Assert.True(FeedTarget(bufferBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true));
Assert.True(FeedTarget(bufferBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true));
Assert.True(FeedTarget(bufferBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true));
Assert.True(FeedTarget(bufferBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true));
Assert.True(FeedTarget(bufferBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true));
}
// Test chained Post/Receive
{
localPassed = true;
const int ITERS = 2;
var network = Chain<BufferBlock<int>, int>(4, () => new BufferBlock<int>());
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i);
}
Assert.True(localPassed, string.Format("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"));
}
// Test chained SendAsync/Receive
{
localPassed = true;
const int ITERS = 2;
var network = Chain<BufferBlock<int>, int>(4, () => new BufferBlock<int>());
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i);
}
Assert.True(localPassed, string.Format("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"));
}
// Test chained Post all then Receive
{
localPassed = true;
const int ITERS = 2;
var network = Chain<BufferBlock<int>, int>(4, () => new BufferBlock<int>());
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i;
Assert.True(localPassed, string.Format("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"));
}
// Test chained SendAsync all then Receive
{
localPassed = true;
const int ITERS = 2;
var network = Chain<BufferBlock<int>, int>(4, () => new BufferBlock<int>());
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2));
Assert.True(localPassed, string.Format("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"));
}
// Test using a precanceled token
{
localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new DataflowBlockOptions { CancellationToken = cts.Token };
var bb = new BufferBlock<int>(dbo);
int ignoredValue;
IList<int> ignoredValues;
localPassed &= bb.LinkTo(new ActionBlock<int>(delegate { })) != null;
localPassed &= bb.SendAsync(42).Result == false;
localPassed &= bb.TryReceiveAll(out ignoredValues) == false;
localPassed &= bb.Post(42) == false;
localPassed &= bb.Count == 0;
localPassed &= bb.TryReceive(out ignoredValue) == false;
localPassed &= bb.Completion != null;
bb.Complete();
}
catch (Exception)
{
localPassed = false;
}
Assert.True(localPassed, string.Format(" {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"));
}
}
public void CancellationTest ()
{
var source1 = new BufferBlock<int> ();
var source2 = new BufferBlock<long> ();
var tokenSource = new CancellationTokenSource ();
var options = new DataflowBlockOptions
{ CancellationToken = tokenSource.Token };
var completion = DataflowBlock.Choose (
source1, _ => { }, source2, _ => { }, options);
Assert.IsFalse (completion.IsCanceled);
tokenSource.Cancel ();
Thread.Sleep (100);
Assert.IsTrue (completion.IsCanceled);
}
public void RunDataflowBlockOptionsTests()
{
// Test base DataflowBlockOptions
{
// Test invalid property values
{
Assert.Throws<ArgumentNullException>(() => { new DataflowBlockOptions().TaskScheduler = null; });
Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().MaxMessagesPerTask = -2; });
Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().MaxMessagesPerTask = 0; });
Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().BoundedCapacity = -2; });
Assert.Throws<ArgumentOutOfRangeException>(() => { new DataflowBlockOptions().BoundedCapacity = 0; });
Assert.Throws<ArgumentNullException>(() => { new DataflowBlockOptions().NameFormat = null; });
}
// Test default values
{
var db = new DataflowBlockOptions();
Assert.True(db.TaskScheduler == TaskScheduler.Default, "TaskScheduler should be Default");
Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Max messages should be unbounded.");
Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Bounded capacity should be unbounded.");
Assert.True(
!db.CancellationToken.CanBeCanceled && !db.CancellationToken.IsCancellationRequested,
"The cancellation token should be None.");
Assert.True(DataflowBlockOptions.Unbounded == -1, "Unbounded should be the value -1");
Assert.True(db.NameFormat == @"{0} Id={1}", @"NameFormat should be the value '{0} Id={1}'");
}
// Test that set values are retrievable
{
var db = new DataflowBlockOptions();
db.MaxMessagesPerTask = 2;
Assert.True(db.MaxMessagesPerTask == 2, "Expected max messages to be the set value 2");
db.MaxMessagesPerTask = Int32.MaxValue;
Assert.True(db.MaxMessagesPerTask == Int32.MaxValue, "Expected max messages to be the set value Int32.MaxValue");
db.MaxMessagesPerTask = DataflowBlockOptions.Unbounded;
Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max messages to be unbounded.");
db.BoundedCapacity = 2;
Assert.True(db.BoundedCapacity == 2, "Expected bounded capacity to be the set value 2");
db.BoundedCapacity = Int32.MaxValue;
Assert.True(db.BoundedCapacity == Int32.MaxValue, "Expected bounded capacity to be the set value Int32.MaxValue");
db.BoundedCapacity = DataflowBlockOptions.Unbounded;
Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Expected bounded capacity to be unbounded.");
var dummyScheduler = new DummyScheduler();
db.TaskScheduler = dummyScheduler;
Assert.True(db.TaskScheduler == dummyScheduler, "Expected task scheduler to be the dummy scheduler");
db.TaskScheduler = TaskScheduler.Default;
Assert.True(db.TaskScheduler == TaskScheduler.Default, "Expected task scheduler to be the default scheduler");
var cts = new CancellationTokenSource();
db.CancellationToken = cts.Token;
Assert.True(db.CancellationToken == cts.Token, "Expected the token to be the one just set");
db.CancellationToken = CancellationToken.None;
Assert.True(db.CancellationToken == CancellationToken.None, "Expected the token to be none");
db.NameFormat = "none";
Assert.True(db.NameFormat.Equals("none"), "Expected name format to be the set value 'none'");
db.NameFormat = "foo {0}";
Assert.True(db.NameFormat.Equals("foo {0}"), @"Expected name format to be the set value 'foo {0}'");
db.NameFormat = "foo {0} bar {1}";
Assert.True(db.NameFormat.Equals("foo {0} bar {1}"), @"Expected name format to be the set value 'foo {0} bar {1}'");
db.NameFormat = "kaboom {0} {1} {2}";
Assert.True(db.NameFormat.Equals("kaboom {0} {1} {2}"), @"Expected name format to be the set value 'kaboom {0} {1} {2}'");
}
}
// Test base ExecutionDataflowBlockOptions
{
// Test invalid property values
{
Assert.Throws<ArgumentOutOfRangeException>(() => { new ExecutionDataflowBlockOptions().MaxDegreeOfParallelism = -2; });
Assert.Throws<ArgumentOutOfRangeException>(() => { new ExecutionDataflowBlockOptions().MaxDegreeOfParallelism = 0; });
}
// Test default values
{
var db = new ExecutionDataflowBlockOptions();
Assert.True(db.TaskScheduler == TaskScheduler.Default, "Expected task scheduler to have default value");
Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max messages to have default value");
Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Expected bounded capacity to have default value");
Assert.True(
!db.CancellationToken.CanBeCanceled && !db.CancellationToken.IsCancellationRequested, "Expected cancellation token to have default value");
Assert.True(db.MaxDegreeOfParallelism == 1, "Expected max dop to have default value");
}
// Test that set values are retrievable
{
var db = new ExecutionDataflowBlockOptions();
db.MaxDegreeOfParallelism = 2;
Assert.True(db.MaxDegreeOfParallelism == 2, "Expected max dop to be 2");
db.MaxDegreeOfParallelism = Int32.MaxValue;
Assert.True(db.MaxDegreeOfParallelism == Int32.MaxValue, "Expected max dop to be Int32.MaxValue");
db.MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded;
Assert.True(db.MaxDegreeOfParallelism == DataflowBlockOptions.Unbounded, "Expected max dop to be unbounded");
}
}
// Test base GroupingDataflowBlockOptions
{
// Test invalid property values
{
Assert.Throws<ArgumentOutOfRangeException>(() => { new GroupingDataflowBlockOptions().MaxNumberOfGroups = -2; });
Assert.Throws<ArgumentOutOfRangeException>(() => { new GroupingDataflowBlockOptions().MaxNumberOfGroups = 0; });
}
// Test default values
{
var db = new GroupingDataflowBlockOptions();
Assert.True(db.TaskScheduler == TaskScheduler.Default, "Expected task scheduler to have default value");
Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max messages to have default value");
Assert.True(db.BoundedCapacity == DataflowBlockOptions.Unbounded, "Expected bounded capacity to have default value");
Assert.True(
!db.CancellationToken.CanBeCanceled && !db.CancellationToken.IsCancellationRequested, "Expected cancellation token to have default value");
Assert.True(db.MaxNumberOfGroups == DataflowBlockOptions.Unbounded, "Expected max groups to have default value");
Assert.True(db.Greedy == true, "Expected greedy to have default value");
}
// Test that set values are retrievable
{
var db = new GroupingDataflowBlockOptions();
db.MaxNumberOfGroups = 2;
Assert.True(db.MaxNumberOfGroups == 2, "Expected max groups to be 2");
db.MaxNumberOfGroups = Int32.MaxValue;
Assert.True(db.MaxNumberOfGroups == Int32.MaxValue, "Expected max groups to be Int32.MaxValue");
db.MaxNumberOfGroups = Int64.MaxValue;
Assert.True(db.MaxNumberOfGroups == Int64.MaxValue, "Expected max groups to be Int64.MaxValue");
db.MaxNumberOfGroups = DataflowBlockOptions.Unbounded;
Assert.True(db.MaxMessagesPerTask == DataflowBlockOptions.Unbounded, "Expected max groups to unbounded");
db.Greedy = true;
Assert.True(db.Greedy == true, "Expected greedy to be true");
db.Greedy = false;
Assert.True(db.Greedy == false, "Expected greedy to be false");
db.Greedy = true;
Assert.True(db.Greedy == true, "Expected greedy to be true");
}
}
}
public static CompletionHelper GetNew (DataflowBlockOptions options)
{
return new CompletionHelper (options);
}
public CompletionHelper (DataflowBlockOptions options)
{
if (options != null && options.CancellationToken != CancellationToken.None)
options.CancellationToken.Register (RequestCancel);
}
public void TestBufferBlockCompletionOrder()
{
const int ITERATIONS = 1000;
for (int iter = 0; iter < ITERATIONS; iter++)
{
var cts = new CancellationTokenSource();
var options = new DataflowBlockOptions() { CancellationToken = cts.Token };
var buffer = new BufferBlock<int>(options);
buffer.Post(1);
cts.Cancel();
try { buffer.Completion.Wait(); }
catch { }
Assert.False(buffer.Count != 0, string.Format("Iteration {0}: Completed before clearing messages.", iter));
}
}
public Task Execute(CancellationToken token)
{
var options = new DataflowBlockOptions {CancellationToken = token};
_buffer = new BufferBlock<long>(options);
var hydrate = new TransformBlock<long, Summoner>(id =>
{
var summoner = _lookup.Hydrate(id);
return summoner;
}, new ExecutionDataflowBlockOptions { CancellationToken = token, MaxDegreeOfParallelism = 2 });
var store = new TransformBlock<Summoner, Summoner>(summoner =>
{
if (summoner != null)
_storage.Store(summoner);
return summoner;
}, new ExecutionDataflowBlockOptions {CancellationToken = token, MaxDegreeOfParallelism = 2});
var crawl = new TransformManyBlock<Summoner, FellowPlayerInfo>(async summoner =>
{
var summoners = new List<FellowPlayerInfo>();
var games = new List<PlayerGameStats>();
if (summoner != null)
{
await _crawler.Crawl(summoner, summoners.Add, games.Add);
}
return summoners;
}, new ExecutionDataflowBlockOptions {CancellationToken = token, MaxDegreeOfParallelism = 2});
var storeNextBatch = new ActionBlock<FellowPlayerInfo>(async info =>
{
if (info != null)
{
var data = await _lookup.Lookup(info.summonerId);
_storage.StoreWhenMissing(data);
}
}, new ExecutionDataflowBlockOptions {CancellationToken = token, MaxDegreeOfParallelism = 2});
_buffer.LinkTo(hydrate, new DataflowLinkOptions {PropagateCompletion = true});
hydrate.LinkTo(store, new DataflowLinkOptions {PropagateCompletion = true});
store.LinkTo(crawl, new DataflowLinkOptions {PropagateCompletion = true});
crawl.LinkTo(storeNextBatch, new DataflowLinkOptions {PropagateCompletion = true});
return Task.Run(async () =>
{
while (!token.IsCancellationRequested)
{
try
{
var batch = _producer.Produce((int) TimeSpan.FromDays(1).TotalMinutes, 30);
foreach (var id in batch)
await _buffer.SendAsync(id, token);
// Start the chain
_buffer.Complete();
// Wait until the chain is complete before iterating again
await storeNextBatch.Completion;
}
catch (Exception ex)
{
throw;
}
}
}, token);
}
/// <summary>
/// Create a logging thread to process the logging queue
/// </summary>
private void CreateLoggingEventQueue()
{
// We are creating a two-node dataflow graph here. The first node is a buffer, which will hold up to the number of
// logging events we have specified as the queueCapacity. The second node is the processor which will actually process each message.
// When the capacity of the buffer is reached, further attempts to send messages to it will block.
// The reason we can't just set the BoundedCapacity on the processing block is that ActionBlock has some weird behavior
// when the queue capacity is reached. Specifically, it will block new messages from being processed until it has
// entirely drained its input queue, as opposed to letting new ones in as old ones are processed. This is logged as
// a perf bug (305575) against Dataflow. If they choose to fix it, we can eliminate the buffer node from the graph.
var dataBlockOptions = new DataflowBlockOptions
{
BoundedCapacity = Convert.ToInt32(_queueCapacity)
};
_loggingQueue = new BufferBlock<object>(dataBlockOptions);
var executionDataBlockOptions = new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1
};
_loggingQueueProcessor = new ActionBlock<object>(loggingEvent => LoggingEventProcessor(loggingEvent), executionDataBlockOptions);
var dataLinkOptions = new DataflowLinkOptions
{
PropagateCompletion = true
};
_loggingQueue.LinkTo(_loggingQueueProcessor, dataLinkOptions);
}
public void ProduceLogs(int count, int buffSize)
{
var bufferOptions = new DataflowBlockOptions() { BoundedCapacity = buffSize, MaxMessagesPerTask = 10 };
var writerOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 10, MaxDegreeOfParallelism = 1, MaxMessagesPerTask = 10 };
var serializerOptions = new ExecutionDataflowBlockOptions() { BoundedCapacity = 80, MaxDegreeOfParallelism = 8, SingleProducerConstrained = true, MaxMessagesPerTask = 10 };
LogGenerator g = new LogGenerator();
var file = new StreamWriter("basic.async.srlz.buff.log", false);
BufferBlock<LogEntry> buffer = new BufferBlock<LogEntry>(bufferOptions);
TransformBlock<LogEntry, string> serializer = new TransformBlock<LogEntry, string>(
e => string.Format(e.format, e.parameters),
serializerOptions);
ActionBlock<string> writer = new ActionBlock<string>(s => file.WriteLine(s), writerOptions);
buffer.LinkTo(serializer, new DataflowLinkOptions() { PropagateCompletion = true });
serializer.LinkTo(writer, new DataflowLinkOptions() { PropagateCompletion = true });
for (int i = 0; i < count; i++)
{
g.Next();
var entry = new LogEntry() { format = g.FormatStr, parameters = new object[] { g.Param1, g.Param2, g.Param3, g.Param4, g.Param5, g.Param6 } };
buffer.SendAsync(entry).Wait();
}
buffer.Complete();
Completed = writer.Completion.ContinueWith(t => file.Close());
}
public ChooserBlock(Action <T1> action1, Action <T2> action2, Action <T3> action3, DataflowBlockOptions dataflowBlockOptions)
{
// TODO: take care of options and its cancellation token
Target1 = new ChooseTarget <T1> (message => MessageArrived(0, action1, message));
Target2 = new ChooseTarget <T2> (message => MessageArrived(1, action2, message));
Target3 = new ChooseTarget <T3> (message => MessageArrived(2, action3, message));
}