public async Task TestTriggerBatch_NonGreedy()
{
var dbo = new GroupingDataflowBlockOptions {
Greedy = false
};
const int BatchSize = 10;
for (int numPostponedMessages = 1; numPostponedMessages < BatchSize; numPostponedMessages++)
{
var b = new BatchBlock <int>(BatchSize, dbo);
Assert.Equal(expected: 0, actual: b.OutputCount);
for (int i = 0; i < numPostponedMessages; i++)
{
Assert.False(b.SendAsync(i).IsCompleted);
}
b.TriggerBatch();
int[] results = await b.ReceiveAsync();
Assert.Equal(expected: numPostponedMessages, actual: results.Length);
for (int i = 0; i < results.Length; i++)
{
Assert.Equal(expected: i, actual: results[i]);
}
Assert.Equal(expected: 0, actual: b.OutputCount);
b.TriggerBatch();
Assert.Equal(expected: 0, actual: b.OutputCount);
}
}
public ExecutionPipeline(Kernel kernel)
{
_kernel = kernel;
_commandQueue = new BufferBlock<CommandRequest[]>();
_queryQueue = new BatchBlock<QueryRequest>(MaxConcurrentQueries);
var transactionHandler = new ActionBlock<object>(t =>
{
if (t is QueryRequest[])
{
var queries = t as QueryRequest[];
Task[] tasks = queries.Select(q => Task.Factory.StartNew(_ => ExecuteQuery(q), null)).ToArray();
Task.WaitAll(tasks);
}
else if (t is CommandRequest[])
{
var commands = t as CommandRequest[];
foreach (var commandContext in commands)
{
var result = _kernel.Execute(commandContext.Command);
commandContext.Response.Post(result);
}
}
});
_commandQueue.LinkTo(transactionHandler);
_queryQueue.LinkTo(transactionHandler);
_timer = new Timer(_ => _queryQueue.TriggerBatch());
_timer.Change(Interval, Interval);
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_log = SuperCheapIOC.Resolve<ILog>();
_systemMetrics = systemMetrics;
var config = new SqlServerConfiguration(configElement.Attribute("connectionString").Value, configElement.ToInt("writeBatchSize"));
_connectionString = config.ConnectionString;
_collectorName = collectorName;
_retries = config.Retries;
InitialiseRetryHandling();
_batchBlock = new BatchBlock<GraphiteLine>(config.WriteBatchSize);
_actionBlock = new ActionBlock<GraphiteLine[]>(p => SendToDB(p), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1 });
_batchBlock.LinkTo(_actionBlock);
_batchBlock.Completion.ContinueWith(p => _actionBlock.Complete());
_actionBlock.Completion.ContinueWith(p => { _isActive = false; });
_completionTask = new Task(() =>
{
_log.Info("SqlServerBackend - Completion has been signaled. Waiting for action block to complete.");
_batchBlock.Complete();
_actionBlock.Completion.Wait();
});
}
public ExecutionPipeline(Kernel kernel)
{
_kernel = kernel;
_commandQueue = new BufferBlock <CommandRequest[]>();
_queryQueue = new BatchBlock <QueryRequest>(MaxConcurrentQueries);
var transactionHandler = new ActionBlock <object>(t =>
{
if (t is QueryRequest[])
{
var queries = t as QueryRequest[];
Task[] tasks = queries.Select(q => Task.Factory.StartNew(_ => ExecuteQuery(q), null)).ToArray();
Task.WaitAll(tasks);
}
else if (t is CommandRequest[])
{
var commands = t as CommandRequest[];
foreach (var commandContext in commands)
{
var result = _kernel.Execute(commandContext.Command);
commandContext.Response.Post(result);
}
}
});
_commandQueue.LinkTo(transactionHandler);
_queryQueue.LinkTo(transactionHandler);
_timer = new Timer(_ => _queryQueue.TriggerBatch());
_timer.Change(Interval, Interval);
}
public JournalActor(IJournalPersistor journalPersistor, long batchDelayMs = 10L, int batchSize = 100)
{
_journalPersistor = journalPersistor;
_journalingBlock = new ActionBlock<IJournalable[]>(journalables =>
{
var journalStatisticEntry = new JournalStatisticEntry {BatchSize = journalables.Length};
journalStatisticEntry.OverallElapsed = StopWatchUtil.Measure(() =>
{
journalStatisticEntry.WritingOnlyElapsed =
StopWatchUtil.Measure(
() => { _journalPersistor.WriteAsync(DateTimeOffset.Now, journalables); });
});
Statistic.Add(journalStatisticEntry);
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1
});
_requestBlock = new BatchBlock<IJournalable>(batchSize);
_requestBlock.LinkTo(_journalingBlock, new DataflowLinkOptions
{
PropagateCompletion = true
});
_timer = new Timer();
_timer.Elapsed += (sender, args) => { _requestBlock.TriggerBatch(); };
_timer.Interval = batchDelayMs;
_timer.Enabled = true;
}
/// <summary>
/// Create loader
/// </summary>
/// <param name="executor"></param>
/// <param name="serializer"></param>
/// <param name="logger"></param>
/// <param name="addOnly"></param>
/// <param name="bulkSize"></param>
internal BulkImporter(IBulkExecutor executor, JsonSerializerSettings serializer,
ILogger logger, bool addOnly = false, int bulkSize = 10000)
{
_executor = executor ?? throw new ArgumentNullException(nameof(executor));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_serializer = serializer == null?
JsonSerializer.CreateDefault() : JsonSerializer.Create(serializer);
_bulkSize = bulkSize;
_addOnly = addOnly;
// Set up batch blocks
_batcher = new BatchBlock <object>(_bulkSize,
new GroupingDataflowBlockOptions());
var importer = new ActionBlock <object[]>(ProcessBatch,
new ExecutionDataflowBlockOptions {
BoundedCapacity = 1,
MaxDegreeOfParallelism = 1,
SingleProducerConstrained = true
});
// Connect the output to the action handler
_batcher.LinkTo(importer, new DataflowLinkOptions {
PropagateCompletion = true
});
// When done, cause end to be called
_complete = _batcher.Completion
.ContinueWith(async t => {
importer.Complete();
// Drain
await importer.Completion;
});
_cts = new CancellationTokenSource();
}
internal static bool BufferBlocksToBatchNonGreedyToAction()
{
const int ITERS = 2;
var inputs = Enumerable.Range(0, 1).Select(_ => new BufferBlock <int>()).ToList();
var b = new BatchBlock <int>(inputs.Count);
int completedCount = 0;
var c = new ActionBlock <int[]>(i => completedCount++);
foreach (var input in inputs)
{
input.LinkTo(b);
}
Task.Factory.ContinueWhenAll(inputs.Select(i => i.Completion).ToArray(), _ => b.Complete());
b.LinkWithCompletion(c);
for (int i = 0; i < ITERS; i++)
{
inputs[i % inputs.Count].Post(i);
}
foreach (var input in inputs)
{
input.Complete();
}
c.Completion.Wait();
return(completedCount == ITERS / b.BatchSize);
}
public async Task TestBatchingFromSubsetOfSources()
{
const int Batches = 5;
foreach (int batchSize in new[] { 1, 2, 5 })
{
foreach (bool greedy in DataflowTestHelpers.BooleanValues)
{
var batch = new BatchBlock <int>(batchSize, new GroupingDataflowBlockOptions {
Greedy = greedy
});
var buffers = Enumerable.Range(0, batchSize * Batches).Select(_ => new BufferBlock <int>()).ToList();
foreach (var buffer in buffers)
{
buffer.LinkTo(batch);
buffer.Post(1);
}
for (int i = 0; i < Batches; i++)
{
Assert.Equal(expected: batchSize, actual: (await batch.ReceiveAsync()).Sum());
}
}
}
}
static void Main(string[] args)
{
var batchBlock = new BatchBlock <int>(3);
for (int i = 0; i < 10; i++)
{
batchBlock.Post(i);
}
batchBlock.Complete();
batchBlock.Post(10);
for (int i = 0; i < 5; i++)
{
if (batchBlock.TryReceive(out var result))
{
Console.Write($"Received batch {i}: ");
foreach (var r in result)
{
Console.Write(r + " ");
}
Console.WriteLine();
}
else
{
Console.WriteLine("block finished");
break;
}
}
Console.WriteLine("done");
Console.ReadKey();
}
public async Task BufferBlocksToBatchNonGreedyToAction()
{
var inputs = Enumerable.Range(0, 1).Select(_ => new BufferBlock <int>()).ToList();
var b = new BatchBlock <int>(inputs.Count);
int completedCount = 0;
var c = new ActionBlock <int[]>(i => completedCount++);
foreach (var input in inputs)
{
input.LinkTo(b);
}
var ignored = Task.WhenAll(inputs.Select(s => s.Completion)).ContinueWith(
_ => b.Complete(), CancellationToken.None, TaskContinuationOptions.None, TaskScheduler.Default);
b.LinkTo(c, new DataflowLinkOptions {
PropagateCompletion = true
});
for (int i = 0; i < Iterations; i++)
{
inputs[i % inputs.Count].Post(i);
}
foreach (var input in inputs)
{
input.Complete();
}
await c.Completion;
Assert.Equal(expected: Iterations / b.BatchSize, actual: completedCount);
}
public void TestTriggerBatch_VaryingBatchSizes()
{
foreach (var batchSize in new[] { 2, 5 })
{
foreach (var queuedBeforeTrigger in new[] { 1, batchSize - 1 })
{
var b = new BatchBlock <int>(batchSize);
b.PostRange(1, queuedBeforeTrigger + 1);
Assert.Equal(expected: 0, actual: b.OutputCount);
b.TriggerBatch();
Assert.Equal(expected: 1, actual: b.OutputCount);
int[] results;
Assert.True(b.TryReceive(out results));
Assert.Equal(expected: queuedBeforeTrigger, actual: results.Length);
for (int j = 0; j < batchSize; j++)
{
Assert.Equal(expected: 0, actual: b.OutputCount);
b.Post(j);
}
Assert.Equal(expected: 1, actual: b.OutputCount);
}
}
}
// A BatchBlock takes in multiple types of values and outputs those values joined together (amount specified in the constructor)
private static async Task SimpleDemoAsync()
{
Console.WriteLine("BatchBlockDemo has started!");
var block = new BatchBlock <int>(3); // batch messages of type int, into 3
for (int i = 0; i < 10; i++)
{
block.Post(i);
}
block.Complete(); // No mo data.
while (await block.OutputAvailableAsync().ConfigureAwait(false))
{
var output = await block.ReceiveAsync().ConfigureAwait(false);
Console.WriteLine($"BatchBlock BatchOutput: {string.Join(",", output)}");
Console.WriteLine($"BatchBlock OutputCount: {block.OutputCount}");
}
// wait for completion.
await block.Completion.ConfigureAwait(false);
Console.WriteLine("Finished!");
Console.ReadKey();
}
public void TriggerBatchTest ()
{
int[] array = null;
var evt = new ManualResetEventSlim (false);
var buffer = new BatchBlock<int> (10);
var block = new ActionBlock<int[]> (i =>
{
array = i;
evt.Set ();
});
buffer.LinkTo (block);
for (int i = 0; i < 9; i++)
Assert.IsTrue (buffer.Post (i));
buffer.TriggerBatch ();
evt.Wait ();
Assert.IsNotNull (array);
Assert.IsTrue (buffer.Post (42));
evt.Wait (1600);
CollectionAssert.AreEquivalent (new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8 },
array);
}
public Task DownloadThemAllAsync(IEnumerable <Uri> uris, ProcessResult processResult, byte maxThreads)
{
//Multiple Consumers
var consumers = new ActionBlock <Uri[]>( //Handlers
uriBatch => ConsumerAsync(uriBatch, processResult), //!SPOT: thread-safety should be guaranteed by us
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxThreads
});
var sharedUris = new BatchBlock <Uri>(2 * maxThreads); //Command
//This is is the link between the pipeline's command and handlers
//!SPOT: Link together producer to consumers
sharedUris.LinkTo(consumers,
new DataflowLinkOptions {
PropagateCompletion = true
}); //!SPOT: without this option the consumers will never finish
//Single Producer
Task.Run(async() => {
await ProducerAsync(sharedUris, uris);
//Signaling producing is over
sharedUris.Complete();
});
//Waiting for all consumers to finish
return(consumers.Completion);
}
public static async Task <List <string> > LoadFiles(ConcurrentDictionary <string, MemoryStream> streams)
{
List <string> _errors = new List <string>();
Queue <KeyValuePair <string, MemoryStream> > files = new Queue <KeyValuePair <string, MemoryStream> >(streams);
var batchBlock = new BatchBlock <KeyValuePair <string, MemoryStream> >(75, new GroupingDataflowBlockOptions {
BoundedCapacity = 100
});
var actionBlock = new ActionBlock <KeyValuePair <string, MemoryStream>[]>(t =>
{
for (int i = 0; i < t.Length; i++)
{
var s = files.Dequeue();
try
{
DBReader reader = new DBReader();
DBEntry entry = reader.Read(s.Value, s.Key);
if (entry != null)
{
var current = Entries.FirstOrDefault(x => x.FileName == entry.FileName && x.Build == entry.Build);
if (current != null)
{
Entries.Remove(current);
}
Entries.Add(entry);
if (!string.IsNullOrWhiteSpace(reader.ErrorMessage))
{
_errors.Add(FormatError(s.Key, ErrorType.Warning, reader.ErrorMessage));
}
}
}
catch (ConstraintException ex) { _errors.Add(FormatError(s.Key, ErrorType.Error, "Id column contains duplicates.")); }
catch (Exception ex) { _errors.Add(FormatError(s.Key, ErrorType.Error, ex.Message)); }
if (i % 100 == 0 && i > 0)
{
ForceGC();
}
}
ForceGC();
});
batchBlock.LinkTo(actionBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
foreach (KeyValuePair <string, MemoryStream> i in streams)
{
await batchBlock.SendAsync(i); // wait synchronously for the block to accept.
}
batchBlock.Complete();
await actionBlock.Completion;
ForceGC();
return(_errors);
}
public void BasicUsageTest ()
{
int[] array = null;
var evt = new ManualResetEventSlim (false);
var buffer = new BatchBlock<int> (10);
var block = new ActionBlock<int[]> (i =>
{
array = i;
evt.Set ();
});
buffer.LinkTo<int[]> (block);
for (int i = 0; i < 9; i++)
Assert.IsTrue (buffer.Post (i));
Assert.IsFalse (evt.Wait (100));
Assert.IsNull (array);
Assert.IsTrue (buffer.Post (42));
Assert.IsTrue (evt.Wait (1000));
Assert.IsNotNull (array);
CollectionAssert.AreEqual (new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 42 }, array);
}
private static IPropagatorBlock <long, long> Create3AverageNode(Func <long[], long> f)
{
const int BATCH_SIZE = 3;
// create
var target = new BatchBlock <long>(BATCH_SIZE);
var source = new TransformBlock <long[], long>(x => f(x)); // f = LongOperationArray
// link
target.LinkTo(source);
// completion
target.Completion.ContinueWith(completion =>
{
if (completion.IsFaulted)
{
((IDataflowBlock)source).Fault(completion.Exception);
}
else
{
source.Complete();
}
});
return(DataflowBlock.Encapsulate(target, source));
}
private static IPropagatorBlock <long, long> Create2CrossNode_2(Func <long[], long> f)
{
const int BATCHSIZE = 2;
// creation
var target = new BatchBlock <long>(BATCHSIZE, new GroupingDataflowBlockOptions()
{
Greedy = true
});
var source = new TransformBlock <long[], long>(x => f(x)); // f = LongOperationArray
// linking
target.LinkTo(source);
// completion
target.Completion.ContinueWith(completion =>
{
if (completion.IsFaulted)
{
((IDataflowBlock)source).Fault(completion.Exception);
}
else
{
source.Complete();
}
});
return(DataflowBlock.Encapsulate(target, source));
}
public static Tuple <IPropagatorBlock <long, long>[], IPropagatorBlock <long[], long> > GenerateSumGraph(int cntNodes,
Func <long[], long> f)
{
Stopwatch sw = new Stopwatch();
sw.Restart();
// creation
IPropagatorBlock <long, long>[] nodes = new BufferBlock <long> [cntNodes];
for (int i = 0; i < cntNodes; i++)
{
nodes[i] = new BufferBlock <long>();
}
BatchBlock <long> batcher = new BatchBlock <long>(cntNodes, new GroupingDataflowBlockOptions()
{
Greedy = true
});
IPropagatorBlock <long[], long> transformer = new TransformBlock <long[], long>(x => f(x)); // f = LongOperationArray
// linking
foreach (var item in nodes)
{
item.LinkTo(batcher);
}
batcher.LinkTo(transformer);
Console.WriteLine("Creation time: {0}", sw.Elapsed);
return(new Tuple <IPropagatorBlock <long, long>[], IPropagatorBlock <long[], long> >(nodes, transformer));
}
public async Task TestMaxNumberOfGroups()
{
foreach (bool greedy in DataflowTestHelpers.BooleanValues)
{
for (int maxNumberOfGroups = 1; maxNumberOfGroups <= 21; maxNumberOfGroups += 20)
{
for (int itemsPerBatch = 1; itemsPerBatch <= 3; itemsPerBatch += 2)
{
var batch = new BatchBlock <int>(itemsPerBatch,
new GroupingDataflowBlockOptions {
MaxNumberOfGroups = maxNumberOfGroups, Greedy = greedy
});
// Feed all N batches; all should succeed
for (int batchNum = 0; batchNum < maxNumberOfGroups; batchNum++)
{
var sendAsyncs = new Task <bool> [itemsPerBatch];
for (int itemNum = 0; itemNum < itemsPerBatch; itemNum++)
{
sendAsyncs[itemNum] = batch.SendAsync(itemNum);
if (greedy)
{
Assert.True(sendAsyncs[itemNum].IsCompleted);
Assert.True(sendAsyncs[itemNum].Result);
}
else if (itemNum < itemsPerBatch - 1)
{
Assert.False(sendAsyncs[itemNum].IsCompleted);
}
}
await Task.WhenAll(sendAsyncs);
}
// Next message should fail in greedy mode
if (greedy)
{
var t = batch.SendAsync(1);
Assert.Equal(expected: TaskStatus.RanToCompletion, actual: t.Status);
Assert.False(t.Result);
}
// Make sure all batches were produced
for (int i = 0; i < maxNumberOfGroups; i++)
{
int[] result = await batch.ReceiveAsync();
Assert.Equal(expected: itemsPerBatch, actual: result.Length);
}
// Next message should fail, even after groups have been produced
if (!greedy)
{
var t = batch.SendAsync(1);
Assert.Equal(expected: TaskStatus.RanToCompletion, actual: t.Status);
Assert.False(t.Result);
}
}
}
}
}
public IPropagatorDataflowBuilder <TInput, TOutput[]> Batch(int batchSize, DataflowBatchOptions batchOptions = default(DataflowBatchOptions))
{
var batchBlock = new BatchBlock <TOutput>(batchSize, batchOptions.BatchBlockOptions);
LinkHelper.Link(_finalSourceBlock, batchBlock, batchOptions.LinkOptions);
return(new PropagatorDataflowBuilder <TInput, TOutput[]>(_originalTargetBlock, _finalSourceBlock, batchBlock, _propagateCompletion));
}
public void TestTriggerBatch_Nop()
{
const int Iters = 2;
var b = new BatchBlock <int>(1);
for (int i = 0; i < Iters; i++)
{
b.Post(i);
int outputCount = b.OutputCount;
Assert.Equal(expected: i + 1, actual: outputCount);
b.TriggerBatch();
Assert.Equal(expected: outputCount, actual: b.OutputCount);
}
b = new BatchBlock <int>(1);
Assert.Equal(expected: 0, actual: b.OutputCount);
for (int i = 0; i < 2; i++)
{
b.TriggerBatch();
}
for (int i = 0; i < 2; i++)
{
b.Complete();
b.TriggerBatch();
}
Assert.Equal(expected: 0, actual: b.OutputCount);
}
public async Task TestLinkToOptions()
{
const int Messages = 2;
foreach (bool append in DataflowTestHelpers.BooleanValues)
{
var bb = new BatchBlock <int>(1);
var values = new int[Messages][];
var targets = new ActionBlock <int[]> [Messages];
for (int i = 0; i < Messages; i++)
{
int slot = i;
targets[i] = new ActionBlock <int[]>(item => values[slot] = item);
bb.LinkTo(targets[i], new DataflowLinkOptions {
MaxMessages = 1, Append = append
});
}
bb.PostRange(0, Messages);
bb.Complete();
await bb.Completion;
for (int i = 0; i < Messages; i++)
{
targets[i].Complete();
await targets[i].Completion;
Assert.Equal(
expected: append ? i : Messages - i - 1,
actual: values[i][0]);
}
}
}
public async Task BatchBlockWillCompleteTarget()
{
BatchBlock <int> bb = new BatchBlock <int>(batchSize: 2,
dataflowBlockOptions: new GroupingDataflowBlockOptions()
{
BoundedCapacity = 3
});
TestTargetBlock <int[]> testTarget = new TestTargetBlock <int[]>();
testTarget.ConsumptionMode = DataflowMessageStatus.Accepted;
bb.LinkTo(testTarget, PropagateCompletion);
// Rapidly send 50 messages
Task.WaitAll(Enumerable.Range(0, 50).Select((i) => bb.SendAsync(i)).ToArray(), BurstTimeout);
bb.Complete();
// Completion should run to successful conclusion
await Task.WhenAny(bb.Completion, Task.Delay(CompletionTimeout));
Assert.Equal(TaskStatus.RanToCompletion, bb.Completion.Status);
// Assumption: BufferBlock should also have completed its target
await Task.WhenAny(testTarget.Completion, Task.Delay(CompletionTimeout));
Assert.Equal(TaskStatus.RanToCompletion, testTarget.Completion.Status);
// Assumption: we should have gotten 25 batches
bool allMessagesReceived = await TaskUtils.PollWaitAsync(() => testTarget.MessagesConsumed.Count == 25, MessageArrivalTimeout);
Assert.True(allMessagesReceived);
}
public async Task TestNonGreedy()
{
var batch = new BatchBlock <int>(2, new GroupingDataflowBlockOptions {
Greedy = false
});
for (int trial = 0; trial < 2; trial++)
{
Task <bool> firstSend = batch.SendAsync(1 + trial);
Assert.False(firstSend.IsCompleted); // should always pass, but due to race might not test what we really want it to
Assert.Equal(expected: 0, actual: batch.OutputCount); // ditto
Task <bool> secondSend = batch.SendAsync(3 + trial);
Assert.Equal(expected: true, actual: await firstSend);
Assert.Equal(expected: true, actual: await secondSend);
Assert.Equal(expected: true, actual: await batch.OutputAvailableAsync());
Assert.Equal(expected: 1, actual: batch.OutputCount);
int[] result = await batch.ReceiveAsync();
Assert.NotNull(result);
Assert.Equal(expected: 2, actual: result.Length);
Assert.Equal(expected: 1 + trial, actual: result[0]);
Assert.Equal(expected: 3 + trial, actual: result[1]);
}
batch.Complete();
await batch.Completion;
}
private static int NumReadsFlushedAsSingles(ReadPair rp, ConcurrentDictionary <string, ReadPair> borderlinePairs,
int numReadsFlushedAsSingles, BatchBlock <ReadPair> lineBuffer)
{
if ((ReadIsNearby(rp.Read1) || ReadIsNearby(rp.Read2)) && !(rp.DontOverlap.HasValue && !rp.DontOverlap.Value) &&
!rp.IsImproper)
{
if (!borderlinePairs.ContainsKey(rp.Name))
{
borderlinePairs[rp.Name] = rp;
}
else
{
numReadsFlushedAsSingles++;
// Wait til we find the mate, then post to the next block
// TODO should we actually post the earlier one to both blocks and let it carry itself over? Unfortunately by the time we get to edge state we've already done pair resolution, so starting with this because it's simpler and probably rare.
// TODO document as limitation
foreach (var aln in rp.GetAlignments())
{
borderlinePairs[rp.Name].AddAlignment(aln);
}
borderlinePairs.Remove(rp.Name, out var pairToPost);
pairToPost.PairStatus = PairStatus.Paired;
lineBuffer.Post(pairToPost);
}
}
else
{
lineBuffer.Post(rp);
}
return(numReadsFlushedAsSingles);
}
public async Task TestGreedyFromPosts()
{
const int Batches = 3;
foreach (int batchSize in new[] { 1, 2, 5 })
{
var batch = new BatchBlock <int>(batchSize);
for (int i = 0; i < Batches * batchSize; i++)
{
Assert.True(batch.Post(i));
Assert.Equal(expected: (i + 1) / batchSize, actual: batch.OutputCount);
}
for (int i = 0; i < Batches; i++)
{
int[] result = await batch.ReceiveAsync();
Assert.Equal(expected: batchSize, actual: result.Length);
Assert.Equal(expected: Batches - (i + 1), actual: batch.OutputCount);
for (int j = 0; j < result.Length - 1; j++)
{
Assert.Equal(result[j] + 1, result[j + 1]);
}
}
}
}
public BatchingBlock(TimeSpan timeSpan, ITargetBlock <T[]> processor,
CancellationToken cancellation = default)
{
_timeSpan = timeSpan;
_batchBlock = new BatchBlock <T>(100, new GroupingDataflowBlockOptions
{
CancellationToken = cancellation,
BoundedCapacity = DataflowBlockOptions.Unbounded
});
_trigger = new Timer(_ =>
{
try
{
_batchBlock.TriggerBatch();
}
catch (Exception)
{
// ignored
}
}, null, Timeout.Infinite, Timeout.Infinite);
_batchBlock.LinkTo(processor);
}
public async Task TestMultipleNonGreedyFromSources()
{
const int Batches = 10;
foreach (int batchSize in new[] { 1, 2, 5 })
{
var batch = new BatchBlock <int>(batchSize, new GroupingDataflowBlockOptions {
Greedy = false
});
var buffers = Enumerable.Range(0, batchSize).Select(_ => new BufferBlock <int>()).ToList();
foreach (var buffer in buffers)
{
buffer.LinkTo(batch);
}
int prevSum = -1;
for (int i = 0; i < Batches; i++)
{
for (int j = 0; j < batchSize; j++)
{
buffers[j].Post(i);
}
int sum = (await batch.ReceiveAsync()).Sum();
Assert.True(sum > prevSum);
prevSum = sum;
}
}
}
// Adds random employee data to the database by using dataflow.
// This method is similar to AddEmployees except that it uses batching
// to add multiple employees to the database at a time.
static void AddEmployeesBatched(string connectionString, int batchSize, int count)
{
// Create a BatchBlock<Employee> that holds several Employee objects and
// then propagates them out as an array.
BatchBlock <Employee> batchEmployees = new BatchBlock <Employee>(batchSize);
// Create an ActionBlock<Employee[]> object that adds multiple
// employee entries to the database.
ActionBlock <Employee[]> insertEmployees = new ActionBlock <Employee[]>(a =>
{
DatabaseUtilities.InsertEmployees(a, connectionString, "AddEmployeesBatched");
});
// Link the batch block to the action block.
batchEmployees.LinkTo(insertEmployees);
// When the batch block completes, set the action block also to complete.
batchEmployees.Completion.ContinueWith(obj =>
{
insertEmployees.Complete();
});
// Post several random Employee objects to the batch block.
PostRandomEmployees(batchEmployees, count);
// Set the batch block to the completed state and wait for
// all insert operations to complete.
batchEmployees.Complete();
insertEmployees.Completion.Wait();
}
public async Task TestFaultingAndCancellation()
{
foreach (bool fault in DataflowTestHelpers.BooleanValues)
{
var cts = new CancellationTokenSource();
var bb = new BatchBlock <int>(1, new GroupingDataflowBlockOptions {
CancellationToken = cts.Token
});
bb.PostRange(0, 4);
Assert.Equal(expected: 0, actual: (await bb.ReceiveAsync())[0]);
Assert.Equal(expected: 1, actual: (await bb.ReceiveAsync())[0]);
if (fault)
{
Assert.Throws <ArgumentNullException>(() => ((IDataflowBlock)bb).Fault(null));
((IDataflowBlock)bb).Fault(new InvalidCastException());
await Assert.ThrowsAsync <InvalidCastException>(() => bb.Completion);
}
else
{
cts.Cancel();
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => bb.Completion);
}
Assert.Equal(expected: 0, actual: bb.OutputCount);
}
}
public Subscriber(MessageConfigure messageConfigure, SemaphoreSlim smp, string channelName)
{
ChannelName = channelName;
DicClientSockets = new ConcurrentDictionary <string, IClient>();
_addRemoveBlock = new ActionBlock <IClient>(client =>
{
ClientPost(client);
});
var sendMsgActionBlock = new ActionBlock <MsgEntity[]>(entity =>
{
var msg = entity.JsonSerialize();
//限制同时发送消息数量，限制带宽；队列限制1W条
//具体配置按带宽调整
smp.Wait();
//Parallel.ForEach(DicClientSockets, new ParallelOptions { MaxDegreeOfParallelism = messageConfigure.ChannelMaxDegreeOfParallelism }, item =>
Parallel.ForEach(DicClientSockets, item =>
{
item.Value.MsgReceive(msg);
Thread.Sleep(messageConfigure.SendMsgSpanTime);
});
smp.Release();
}, new ExecutionDataflowBlockOptions {
BoundedCapacity = messageConfigure.BoundedCapacity
});
// 合并5条后发送
_sendMsgBatchBlock = new BatchBlock <MsgEntity>(5);
_sendMsgBatchBlock.LinkTo(sendMsgActionBlock);
}
public ISourceDataflowBuilder <TOutput[]> Batch(int batchSize, DataflowBatchOptions batchOptions)
{
var batchBlock = new BatchBlock <TOutput>(batchSize, batchOptions.BatchBlockOptions);
LinkHelper.Link(_finalSourceBlock, batchBlock, batchOptions.LinkOptions);
return(new SourceDataflowBuilder <TOutput[]>(_originalSourceBlock, _finalSourceBlock, batchBlock, _propagateCompletion));
}
static void ShowBatchBlock()
{
// <snippet7>
// Create a BatchBlock<int> object that holds ten
// elements per batch.
var batchBlock = new BatchBlock <int>(10);
// Post several values to the block.
for (int i = 0; i < 13; i++)
{
batchBlock.Post(i);
}
// Set the block to the completed state. This causes
// the block to propagate out any remaining
// values as a final batch.
batchBlock.Complete();
// Print the sum of both batches.
Console.WriteLine("The sum of the elements in batch 1 is {0}.",
batchBlock.Receive().Sum());
Console.WriteLine("The sum of the elements in batch 2 is {0}.",
batchBlock.Receive().Sum());
/* Output:
* The sum of the elements in batch 1 is 45.
* The sum of the elements in batch 2 is 33.
*/
// </snippet7>
}
/// <summary>
/// GeneratePipeline_DataFlowSource_to_DataFlowDestination generates a TPL-DataFlowPipeline between two vertices of a graph.
/// v_source.UserDefinedObjects[0] has to be Type of IDataFlowSource
/// v_dest.UserDefinedObjects[0] has to be Type of IDataFlowDestination
/// </summary>
/// <param name="v_source"></param>
/// <param name="v_dest"></param>
/// <param name="ToCompleteCollection"></param>
/// <param name="WatingForCompletitionCollection"></param>
private void GeneratePipeline_DataFlowSource_to_DataFlowDestination(Vertex v_source, Vertex v_dest, ref List <object> ToCompleteCollection, ref List <object> WatingForCompletitionCollection, ref Dictionary <IDataFlowSource <DS>, object> DataFlowReaderCollection)
{
IDataFlowSource <DS> t_b_source = (IDataFlowSource <DS>)v_source.UserDefinedObjects[0];
IDataFlowDestination <DS> dest = (IDataFlowDestination <DS>)v_dest.UserDefinedObjects[0];
TransformBlock <DS, DS> t_b_dummy = new TransformBlock <DS, DS>(DS => { return(DS); }
, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = this.MaxDegreeOfParallelism
});
ToCompleteCollection.Add(t_b_dummy);
v_dest.UserDefinedObjects.Add(t_b_dummy);
DataFlowReaderCollection.Add(t_b_source, t_b_dummy);
var bacthBlock = new BatchBlock <DS>(BatchSize);
var DataFlowDestinationBlock = new ActionBlock <DS[]>(outp => dest.WriteBatch(outp));
t_b_dummy.LinkTo(bacthBlock, linkOptions);
bacthBlock.LinkTo(DataFlowDestinationBlock, linkOptions);
t_b_dummy.Completion.ContinueWith(t => { bacthBlock.Complete(); });
bacthBlock.Completion.ContinueWith(t => { DataFlowDestinationBlock.Complete(); });
WatingForCompletitionCollection.Add(DataFlowDestinationBlock);
}
public TplBatchingJournaler(IJournalWriter journalWriter, ExecutionPipeline dispatcher, int batchSize)
{
Interval = TimeSpan.FromMilliseconds(16);
_journalWriter = journalWriter;
_dispatcher = dispatcher;
_writerBlock = new ActionBlock<CommandRequest[]>(batch => Go(batch));
_requestQueue = new BatchBlock<CommandRequest>(batchSize);
_requestQueue.LinkTo(_writerBlock);
}
public void AddWriter(IWriteInterface writer)
{
var batchBlock = new BatchBlock<ValueList>(int.MaxValue);
var timer = new Timer(state => batchBlock.TriggerBatch());
timer.Change(0, 10000);
broadcast.LinkTo(batchBlock);
batchBlock.LinkTo(new ActionBlock<ValueList[]>(vl => writer.Write(vl)));
}
public void TestBatchBlockConstructor()
{
// size without decline without option
var block = new BatchBlock<int>(42);
Assert.False(block.BatchSize != 42, "Constructor failed! BatchSize doesn't match for a brand new BatchBlock.");
// size with decline without option
block = new BatchBlock<int>(43, new GroupingDataflowBlockOptions { MaxNumberOfGroups = 1 });
Assert.False(block.BatchSize != 43, "Constructor failed! BatchSize doesn't match for a brand new BatchBlock.");
// size with decline with not cancelled token and default scheduler
block = new BatchBlock<int>(44, new GroupingDataflowBlockOptions { MaxMessagesPerTask = 1 });
Assert.False(block.BatchSize != 44, "Constructor failed! BatchSize doesn't match for a brand new BatchBlock.");
//with a cancelled token and default scheduler
var token = new CancellationToken(true);
block = new BatchBlock<int>(45, new GroupingDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token, MaxNumberOfGroups = 1 });
Assert.False(block.BatchSize != 45, "Constructor failed! BatchSize doesn't match for a brand new BatchBlock.");
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_completionTask = new Task(() => IsActive = false);
_log = SuperCheapIOC.Resolve<ILog>();
_systemMetrics = systemMetrics;
var config = new LibratoBackendConfiguration(
email: configElement.Attribute("email").Value,
token: configElement.Attribute("token").Value,
numRetries: configElement.ToInt("numRetries"),
retryDelay: Utility.ConvertToTimespan(configElement.Attribute("retryDelay").Value),
postTimeout: Utility.ConvertToTimespan(configElement.Attribute("postTimeout").Value),
maxBatchSize: configElement.ToInt("maxBatchSize"),
countersAsGauges: configElement.ToBoolean("countersAsGauges")
);
_config = config;
_source = collectorName;
_serviceVersion = Assembly.GetEntryAssembly().GetName().Version.ToString();
_preprocessorBlock = new ActionBlock<Bucket>(bucket => ProcessBucket(bucket), Utility.UnboundedExecution());
_batchBlock = new BatchBlock<LibratoMetric>(_config.MaxBatchSize);
_outputBlock = new ActionBlock<LibratoMetric[]>(lines => PostToLibrato(lines), Utility.OneAtATimeExecution());
_batchBlock.LinkTo(_outputBlock);
_client = new RestClient(LIBRATO_API_URL);
_client.Authenticator = new HttpBasicAuthenticator(_config.Email, _config.Token);
_client.Timeout = (int)_config.PostTimeout.TotalMilliseconds;
_retryPolicy = new RetryPolicy<LibratoErrorDetectionStrategy>(_config.NumRetries);
_retryPolicy.Retrying += (sender, args) =>
{
_log.Warn(String.Format("Retry {0} failed. Trying again. Delay {1}, Error: {2}", args.CurrentRetryCount, args.Delay, args.LastException.Message), args.LastException);
_systemMetrics.LogCount("backends.librato.retry");
};
_retryStrategy = new Incremental(_config.NumRetries, _config.RetryDelay, TimeSpan.FromSeconds(2));
IsActive = true;
}
public static Task GenerateAnnotatedPathsAsync(IEnumerable<CollectionPath> paths, Func<FileInfo, bool> filePredicate,
ITargetBlock<AnnotatedPath[]> filePathTargetBlock, CancellationToken cancellationToken)
{
var shuffleBlock = new TransformBlock<AnnotatedPath[], AnnotatedPath[]>(
filenames =>
{
// Sequential names tend to fall into the same AWS S3 partition, so we
// shuffle things around.
RandomUtil.Shuffle(filenames);
return filenames;
}, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = Environment.ProcessorCount });
shuffleBlock.LinkTo(filePathTargetBlock, new DataflowLinkOptions { PropagateCompletion = true });
var batcher = new BatchBlock<AnnotatedPath>(2048, new GroupingDataflowBlockOptions { CancellationToken = cancellationToken });
batcher.LinkTo(shuffleBlock, new DataflowLinkOptions
{
PropagateCompletion = true
});
return PostAllFilePathsAsync(paths, filePredicate, batcher, cancellationToken);
}
public async Task TestBatchingFromSubsetOfSources()
{
const int Batches = 5;
foreach (int batchSize in new[] { 1, 2, 5 })
foreach (bool greedy in DataflowTestHelpers.BooleanValues)
{
var batch = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions { Greedy = greedy });
var buffers = Enumerable.Range(0, batchSize * Batches).Select(_ => new BufferBlock<int>()).ToList();
foreach (var buffer in buffers)
{
buffer.LinkTo(batch);
buffer.Post(1);
}
for (int i = 0; i < Batches; i++)
{
Assert.Equal(expected: batchSize, actual: (await batch.ReceiveAsync()).Sum());
}
}
}
public async Task TestNonGreedyLostMessages()
{
foreach (int batchSize in new[] { 2, 5 })
{
var batch = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions { Greedy = false });
var buffers = Enumerable.Range(0, batchSize - 1).Select(_ => new BufferBlock<int>()).ToList();
var tcs = new TaskCompletionSource<bool>();
int remaining = buffers.Count;
// Offer the batch almost all the messages it needs, but have them consumed by someone else
foreach (var buffer in buffers)
{
buffer.LinkTo(batch);
buffer.LinkTo(new ActionBlock<int>(i => {
if (Interlocked.Decrement(ref remaining) == 0)
{
tcs.SetResult(true);
}
}));
buffer.Post(42);
}
await tcs.Task;
// Now offer from another set of sources that won't lose them
buffers = Enumerable.Range(0, batchSize).Select(_ => new BufferBlock<int>()).ToList();
foreach (var buffer in buffers)
{
buffer.LinkTo(batch);
buffer.Post(43);
buffer.Complete();
}
// Wait until all the messages are consumed
await Task.WhenAll(from buffer in buffers select buffer.Completion);
int[] results = await batch.ReceiveAsync();
Assert.Equal(expected: 0, actual: batch.OutputCount);
batch.Complete();
await batch.Completion;
}
}
public async Task TestLinkToOptions()
{
const int Messages = 2;
foreach (bool append in DataflowTestHelpers.BooleanValues)
{
var bb = new BatchBlock<int>(1);
var values = new int[Messages][];
var targets = new ActionBlock<int[]>[Messages];
for (int i = 0; i < Messages; i++)
{
int slot = i;
targets[i] = new ActionBlock<int[]>(item => values[slot] = item);
bb.LinkTo(targets[i], new DataflowLinkOptions { MaxMessages = 1, Append = append });
}
bb.PostRange(0, Messages);
bb.Complete();
await bb.Completion;
for (int i = 0; i < Messages; i++)
{
targets[i].Complete();
await targets[i].Completion;
Assert.Equal(
expected: append ? i : Messages - i - 1,
actual: values[i][0]);
}
}
}
public async Task TestFaultyScheduler()
{
var bb = new BatchBlock<int>(2, new GroupingDataflowBlockOptions
{
Greedy = false,
TaskScheduler = new DelegateTaskScheduler
{
QueueTaskDelegate = delegate { throw new FormatException(); }
}
});
Task<bool> t1 = bb.SendAsync(1);
Task<bool> t2 = bb.SendAsync(2);
await Assert.ThrowsAsync<TaskSchedulerException>(() => bb.Completion);
Assert.False(await t1);
Assert.False(await t2);
}
public async Task TestFaultingAndCancellation()
{
foreach (bool fault in DataflowTestHelpers.BooleanValues)
{
var cts = new CancellationTokenSource();
var bb = new BatchBlock<int>(1, new GroupingDataflowBlockOptions { CancellationToken = cts.Token });
bb.PostRange(0, 4);
Assert.Equal(expected: 0, actual: (await bb.ReceiveAsync())[0]);
Assert.Equal(expected: 1, actual: (await bb.ReceiveAsync())[0]);
if (fault)
{
Assert.Throws<ArgumentNullException>(() => ((IDataflowBlock)bb).Fault(null));
((IDataflowBlock)bb).Fault(new InvalidCastException());
await Assert.ThrowsAsync<InvalidCastException>(() => bb.Completion);
}
else
{
cts.Cancel();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => bb.Completion);
}
Assert.Equal(expected: 0, actual: bb.OutputCount);
}
}
public void TestTriggerBatch_Cancellation()
{
foreach (bool post in DataflowTestHelpers.BooleanValues)
{
var cts = new CancellationTokenSource();
var b = new BatchBlock<int>(2, new GroupingDataflowBlockOptions { CancellationToken = cts.Token });
Assert.Equal(expected: 0, actual: b.OutputCount);
if (post)
{
b.Post(42);
Assert.Equal(expected: 0, actual: b.OutputCount);
}
cts.Cancel();
Assert.Equal(expected: 0, actual: b.OutputCount);
b.TriggerBatch();
Assert.Equal(expected: 0, actual: b.OutputCount);
}
}
public void TestTriggerBatch_VaryingBatchSizes()
{
foreach (var batchSize in new[] { 2, 5 })
foreach (var queuedBeforeTrigger in new[] { 1, batchSize - 1 })
{
var b = new BatchBlock<int>(batchSize);
b.PostRange(1, queuedBeforeTrigger + 1);
Assert.Equal(expected: 0, actual: b.OutputCount);
b.TriggerBatch();
Assert.Equal(expected: 1, actual: b.OutputCount);
int[] results;
Assert.True(b.TryReceive(out results));
Assert.Equal(expected: queuedBeforeTrigger, actual: results.Length);
for (int j = 0; j < batchSize; j++)
{
Assert.Equal(expected: 0, actual: b.OutputCount);
b.Post(j);
}
Assert.Equal(expected: 1, actual: b.OutputCount);
}
}
public void TestTriggerBatch_Nop()
{
const int Iters = 2;
var b = new BatchBlock<int>(1);
for (int i = 0; i < Iters; i++)
{
b.Post(i);
int outputCount = b.OutputCount;
Assert.Equal(expected: i + 1, actual: outputCount);
b.TriggerBatch();
Assert.Equal(expected: outputCount, actual: b.OutputCount);
}
b = new BatchBlock<int>(1);
Assert.Equal(expected: 0, actual: b.OutputCount);
for (int i = 0; i < 2; i++)
{
b.TriggerBatch();
}
for (int i = 0; i < 2; i++)
{
b.Complete();
b.TriggerBatch();
}
Assert.Equal(expected: 0, actual: b.OutputCount);
}
public async Task TestReleaseOnReserveException()
{
foreach (bool linkBadFirst in DataflowTestHelpers.BooleanValues)
{
var goodSource = new BufferBlock<int>();
goodSource.Post(1);
DelegatePropagator<int, int> badSource = null;
badSource = new DelegatePropagator<int, int>
{
LinkToDelegate = (target, options) => {
target.OfferMessage(new DataflowMessageHeader(1), 2, badSource, consumeToAccept: true);
return new DelegateDisposable();
},
ReserveMessageDelegate = delegate { throw new InvalidCastException(); }
};
var batch = new BatchBlock<int>(2, new GroupingDataflowBlockOptions { Greedy = false });
if (linkBadFirst) // Each linking will offer a message
{
badSource.LinkTo(batch);
goodSource.LinkTo(batch);
}
else
{
goodSource.LinkTo(batch);
badSource.LinkTo(batch);
}
await Assert.ThrowsAnyAsync<InvalidCastException>(() => batch.Completion);
int item;
Assert.True(goodSource.TryReceive(out item)); // The good message must not be Reserved
}
}
public async Task TestMaxNumberOfGroups()
{
foreach (bool greedy in DataflowTestHelpers.BooleanValues)
for (int maxNumberOfGroups = 1; maxNumberOfGroups <= 21; maxNumberOfGroups += 20)
{
for (int itemsPerBatch = 1; itemsPerBatch <= 3; itemsPerBatch += 2)
{
var batch = new BatchBlock<int>(itemsPerBatch,
new GroupingDataflowBlockOptions { MaxNumberOfGroups = maxNumberOfGroups, Greedy = greedy });
// Feed all N batches; all should succeed
for (int batchNum = 0; batchNum < maxNumberOfGroups; batchNum++)
{
var sendAsyncs = new Task<bool>[itemsPerBatch];
for (int itemNum = 0; itemNum < itemsPerBatch; itemNum++)
{
sendAsyncs[itemNum] = batch.SendAsync(itemNum);
if (greedy)
{
Assert.True(sendAsyncs[itemNum].IsCompleted);
Assert.True(sendAsyncs[itemNum].Result);
}
else if (itemNum < itemsPerBatch - 1)
{
Assert.False(sendAsyncs[itemNum].IsCompleted);
}
}
await Task.WhenAll(sendAsyncs);
}
// Next message should fail in greedy mode
if (greedy)
{
var t = batch.SendAsync(1);
Assert.Equal(expected: TaskStatus.RanToCompletion, actual: t.Status);
Assert.False(t.Result);
}
// Make sure all batches were produced
for (int i = 0; i < maxNumberOfGroups; i++)
{
int[] result = await batch.ReceiveAsync();
Assert.Equal(expected: itemsPerBatch, actual: result.Length);
}
// Next message should fail, even after groups have been produced
if (!greedy)
{
var t = batch.SendAsync(1);
Assert.Equal(expected: TaskStatus.RanToCompletion, actual: t.Status);
Assert.False(t.Result);
}
}
}
}
public async Task TestCompletionWithBufferedItems()
{
var b = new BatchBlock<int>(5);
b.PostRange(0, 3);
b.Complete();
await b.OutputAvailableAsync();
Assert.Equal(expected: 1, actual: b.OutputCount);
int[] items = await b.ReceiveAsync();
Assert.Equal(expected: 3, actual: items.Length);
await b.Completion;
}
public async Task TestNonGreedyFailedConsume()
{
foreach (bool exceptionalConsume in DataflowTestHelpers.BooleanValues)
foreach (bool linkGoodFirst in DataflowTestHelpers.BooleanValues)
{
const int BatchSize = 2;
var bb = new BatchBlock<int>(BatchSize, new GroupingDataflowBlockOptions { Greedy = false });
var goodSource = new BufferBlock<int>();
Assert.True(goodSource.Post(1));
if (linkGoodFirst)
{
goodSource.LinkTo(bb);
}
var badSource = new DelegatePropagator<int, int>
{
ReserveMessageDelegate = delegate { return true; },
ConsumeMessageDelegate = delegate(DataflowMessageHeader header, ITargetBlock<int> target, out bool messageConsumed) {
if (exceptionalConsume)
{
throw new FormatException(); // throw when attempting to consume reserved message
}
else
{
messageConsumed = false; // fail when attempting to consume reserved message
return 0;
}
}
};
Assert.Equal(
expected: DataflowMessageStatus.Postponed,
actual: ((ITargetBlock<int>)bb).OfferMessage(new DataflowMessageHeader(2), 2, badSource, consumeToAccept: true));
if (!linkGoodFirst)
{
goodSource.LinkTo(bb);
}
await (exceptionalConsume ?
(Task)Assert.ThrowsAsync<FormatException>(() => bb.Completion) :
(Task)Assert.ThrowsAsync<InvalidOperationException>(() => bb.Completion));
}
}
public async Task TestBoundedCapacityFromSends()
{
foreach (int boundedCapacity in new[] { DataflowBlockOptions.Unbounded, 3 })
foreach (bool greedy in DataflowTestHelpers.BooleanValues)
{
var bb = new BatchBlock<int>(2, new GroupingDataflowBlockOptions { BoundedCapacity = boundedCapacity, Greedy = greedy });
Task<bool>[] sends = Enumerable.Range(0, 100).Select(i => bb.SendAsync(i)).ToArray();
((IDataflowBlock)bb).Fault(new InvalidCastException());
await Assert.ThrowsAsync<InvalidCastException>(() => bb.Completion);
await Task.WhenAll(sends);
}
}
public void TestTriggerBatch_MaxNumberOfGroups()
{
foreach (int maxGroups in new[] { 1, 3 })
{
var b = new BatchBlock<int>(2, new GroupingDataflowBlockOptions { MaxNumberOfGroups = maxGroups });
for (int i = 0; i < maxGroups; i++)
{
b.Post(42);
Assert.Equal(expected: i, actual: b.OutputCount);
b.TriggerBatch();
Assert.Equal(expected: i + 1, actual: b.OutputCount);
}
Assert.False(b.Post(43));
b.TriggerBatch();
Assert.Equal(expected: maxGroups, actual: b.OutputCount);
}
}
public void TestTriggerBatch_NonGreedyEmpty()
{
var dbo = new GroupingDataflowBlockOptions { Greedy = false };
var b = new BatchBlock<int>(3, dbo);
Assert.Equal(expected: 0, actual: b.OutputCount);
b.TriggerBatch();
Assert.Equal(expected: 0, actual: b.OutputCount);
}
public async Task TestTriggerBatch_NonGreedy()
{
var dbo = new GroupingDataflowBlockOptions { Greedy = false };
const int BatchSize = 10;
for (int numPostponedMessages = 1; numPostponedMessages < BatchSize; numPostponedMessages++)
{
var b = new BatchBlock<int>(BatchSize, dbo);
Assert.Equal(expected: 0, actual: b.OutputCount);
for (int i = 0; i < numPostponedMessages; i++)
{
Assert.False(b.SendAsync(i).IsCompleted);
}
b.TriggerBatch();
int[] results = await b.ReceiveAsync();
Assert.Equal(expected: numPostponedMessages, actual: results.Length);
for (int i = 0; i < results.Length; i++)
{
Assert.Equal(expected: i, actual: results[i]);
}
Assert.Equal(expected: 0, actual: b.OutputCount);
b.TriggerBatch();
Assert.Equal(expected: 0, actual: b.OutputCount);
}
}
public async Task TestGreedyFromPosts()
{
const int Batches = 3;
foreach (int batchSize in new[] { 1, 2, 5 })
{
var batch = new BatchBlock<int>(batchSize);
for (int i = 0; i < Batches * batchSize; i++)
{
Assert.True(batch.Post(i));
Assert.Equal(expected: (i+1) / batchSize, actual: batch.OutputCount);
}
for (int i = 0; i < Batches; i++)
{
int[] result = await batch.ReceiveAsync();
Assert.Equal(expected: batchSize, actual: result.Length);
Assert.Equal(expected: Batches - (i + 1), actual: batch.OutputCount);
for (int j = 0; j < result.Length - 1; j++)
{
Assert.Equal(result[j] + 1, result[j + 1]);
}
}
}
}
public async Task TestPrecancellation()
{
var b = new BatchBlock<int>(42, new GroupingDataflowBlockOptions {
CancellationToken = new CancellationToken(canceled: true), MaxNumberOfGroups = 1
});
Assert.Equal(expected: 42, actual: b.BatchSize);
Assert.NotNull(b.LinkTo(DataflowBlock.NullTarget<int[]>()));
Assert.False(b.Post(42));
Task<bool> t = b.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
int[] ignoredValue;
IList<int[]> ignoredValues;
Assert.False(b.TryReceive(out ignoredValue));
Assert.False(b.TryReceiveAll(out ignoredValues));
Assert.Equal(expected: 0, actual: b.OutputCount);
Assert.NotNull(b.Completion);
b.Complete(); // verify doesn't throw
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => b.Completion);
}
public async Task TestNonGreedy()
{
var batch = new BatchBlock<int>(2, new GroupingDataflowBlockOptions { Greedy = false });
for (int trial = 0; trial < 2; trial++)
{
Task<bool> firstSend = batch.SendAsync(1 + trial);
Assert.False(firstSend.IsCompleted); // should always pass, but due to race might not test what we really want it to
Assert.Equal(expected: 0, actual: batch.OutputCount); // ditto
Task<bool> secondSend = batch.SendAsync(3 + trial);
Assert.Equal(expected: true, actual: await firstSend);
Assert.Equal(expected: true, actual: await secondSend);
Assert.Equal(expected: true, actual: await batch.OutputAvailableAsync());
Assert.Equal(expected: 1, actual: batch.OutputCount);
int[] result = await batch.ReceiveAsync();
Assert.NotNull(result);
Assert.Equal(expected: 2, actual: result.Length);
Assert.Equal(expected: 1 + trial, actual: result[0]);
Assert.Equal(expected: 3 + trial, actual: result[1]);
}
batch.Complete();
await batch.Completion;
}
public async Task TestMultipleNonGreedyFromSources()
{
const int Batches = 10;
foreach (int batchSize in new[] { 1, 2, 5 })
{
var batch = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions { Greedy = false });
var buffers = Enumerable.Range(0, batchSize).Select(_ => new BufferBlock<int>()).ToList();
foreach (var buffer in buffers)
{
buffer.LinkTo(batch);
}
int prevSum = -1;
for (int i = 0; i < Batches; i++)
{
for (int j = 0; j < batchSize; j++)
{
buffers[j].Post(i);
}
int sum = (await batch.ReceiveAsync()).Sum();
Assert.True(sum > prevSum);
prevSum = sum;
}
}
}
