/// <summary>
/// 阻塞等待
/// </summary>
public void Complete(bool iswait = false)
{
_batchBlock.Complete();
_batchBlock.Completion.Wait();
_actionBlock.Complete();
if (iswait)
{
_actionBlock.Completion.Wait();
}
}
public IDictionary <string, string[]> BatchBlockUsage(int numberOfIteration, int batchsize)
{
Console.WriteLine($"Inside {nameof(TplDataflow3GroupingBlocksController)} - {nameof(BatchBlockUsage)}");
var ouputCollection = new Dictionary <string, string[]>();
Functions.ClearCounterForBatchBlockUsage();
// Create the members of the pipeline.
var batchBlockWithSizeGivenInInput = new BatchBlock <string>(batchsize);
var actionBlockPerformActionOnBatchData = new ActionBlock <string[]>(batchedInput =>
Functions.DisplayByGroups(ouputCollection, batchedInput)
);
// Connect the dataflow blocks to form a pipeline.
batchBlockWithSizeGivenInInput.LinkTo(actionBlockPerformActionOnBatchData, DataflowOptions.LinkOptions);
// Start BatchBlockUsage pipeline with the input values.
for (var i = 0; i < numberOfIteration; i++)
{
batchBlockWithSizeGivenInInput.Post($"Value = {i}");
}
// Mark the head of the pipeline as complete.
batchBlockWithSizeGivenInInput.Complete();
// Wait for the last block in the pipeline to process all messages.
actionBlockPerformActionOnBatchData.Completion.Wait();
return(ouputCollection);
}
public void ActionQueueTestInPipeline()
{
var list = new List <List <int> >();
var option = new DataflowLinkOptions
{
PropagateCompletion = true,
};
var batchBlock = new BatchBlock <int>(10);
var actionBlock = new ActionBlock <int[]>(x => list.Add(new List <int>(x)));
batchBlock.LinkTo(actionBlock, option);
var p = new PipelineManager <IWorkContext, int>
{
new Pipeline <IWorkContext, int>() + ((c, x) => { batchBlock.Post(x); return(true); }),
};
Enumerable.Range(0, 100)
.ForEach(x => p.Post(null !, x));
batchBlock.Complete();
Task.WaitAll(batchBlock.Completion, actionBlock.Completion);
list.Count.Should().Be(10);
list.All(x => x.Count == 10).Should().BeTrue();
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_log = SuperCheapIOC.Resolve<ILog>();
_systemMetrics = systemMetrics;
var config = new SqlServerConfiguration(configElement.Attribute("connectionString").Value, configElement.ToInt("writeBatchSize"));
_connectionString = config.ConnectionString;
_collectorName = collectorName;
_retries = config.Retries;
InitialiseRetryHandling();
_batchBlock = new BatchBlock<GraphiteLine>(config.WriteBatchSize);
_actionBlock = new ActionBlock<GraphiteLine[]>(p => SendToDB(p), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1 });
_batchBlock.LinkTo(_actionBlock);
_batchBlock.Completion.ContinueWith(p => _actionBlock.Complete());
_actionBlock.Completion.ContinueWith(p => { _isActive = false; });
_completionTask = new Task(() =>
{
_log.Info("SqlServerBackend - Completion has been signaled. Waiting for action block to complete.");
_batchBlock.Complete();
_actionBlock.Completion.Wait();
});
}
/// <summary>
/// Inserts records of the appropriate type to the database
/// </summary>
/// <param name="broadcast"></param>
/// <param name="batchSize"></param>
/// <param name="context"></param>
public static void Insert(BatchBlock <TEntity> broadcast, KeepaContext context)
{
// Create a BatchBlock<best_sellers> that holds several best_seller objects and
// then propagates them out as an array.
//var batchRecs = new BatchBlock<TEntity>(1000);
//var queue = new BufferBlock<best_sellers>();
// Create an ActionBlock<best_seller[]> object that adds multiple
// best_seller entries to the database.
var insertEmployees = new ActionBlock <TEntity[]>(a =>
context.Set <TEntity>().AddRange(a)
);
//Link broadcast to batch
broadcast.LinkTo(insertEmployees, new DataflowLinkOptions {
PropagateCompletion = true
});
// Link the batch block to the action block.
//batchRecs.LinkTo(insertEmployees, new DataflowLinkOptions { PropagateCompletion = true });
// When the batch block completes, set the action block also to complete.
broadcast.Completion.ContinueWith(delegate { insertEmployees.Complete(); });
// Set the batch block to the completed state and wait for
// all insert operations to complete.
broadcast.Complete();
insertEmployees.Completion.Wait();
}
static void Main(string[] args)
{
Task.Run(async() =>
{
bufferBlock = new BufferBlock <string>();
var batchBlock = new BatchBlock <string>(7);
var action = new ActionBlock <IEnumerable <string> >(async t =>
{
await SendUsers(t);
});
batchBlock.LinkTo(action, new DataflowLinkOptions()
{
PropagateCompletion = true
});
bufferBlock.LinkTo(batchBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
await InsertUsers();
Console.WriteLine("Now the buffer will complete. It's complition will be followed by completing the batch. Press any key to continue...");
Console.ReadKey();
bufferBlock.Complete();
await bufferBlock.Completion.ContinueWith(delegate
{
batchBlock.Complete();
});
Console.ReadKey();
}).GetAwaiter().GetResult();
}
static void Main(string[] args)
{
var batchBlock = new BatchBlock <int>(3);
for (int i = 0; i < 10; i++)
{
batchBlock.Post(i);
}
batchBlock.Complete();
batchBlock.Post(99); // This will be ignored
for (int i = 0; i < 5; i++)
{
if (batchBlock.TryReceive(out int[] result))
{
Console.Write($"Received batch {i}: ");
foreach (var item in result)
{
Console.Write($"{item} ");
}
Console.WriteLine();
}
else
{
Console.WriteLine("The block completed.");
break;
}
}
Console.WriteLine("Finished!");
Console.ReadKey();
}
private static void TestBatchBlock()
{
//var batchBlock = new BatchBlock<int>(10);
//for (int i = 0; i < 13; i++)
//{
// batchBlock.Post(i);
//}
//batchBlock.Complete();
//Console.WriteLine("The sum of the elements in batch 1 is {0}.", batchBlock.Receive().Sum());
//Console.WriteLine("The sum of the elements in batch 2 is {0}.", batchBlock.Receive().Sum());
var bb = new BatchBlock <int>(3);
var ab = new ActionBlock <int[]>(i =>
{
string s = string.Empty;
foreach (int m in i)
{
s += m + " ";
}
Console.WriteLine(s);
});
bb.LinkTo(ab);
for (int i = 0; i < 10; i++)
{
bb.Post(i);
}
bb.Complete();
}
public static async Task <List <string> > LoadFiles(ConcurrentDictionary <string, MemoryStream> streams)
{
List <string> _errors = new List <string>();
Queue <KeyValuePair <string, MemoryStream> > files = new Queue <KeyValuePair <string, MemoryStream> >(streams);
var batchBlock = new BatchBlock <KeyValuePair <string, MemoryStream> >(75, new GroupingDataflowBlockOptions {
BoundedCapacity = 100
});
var actionBlock = new ActionBlock <KeyValuePair <string, MemoryStream>[]>(t =>
{
for (int i = 0; i < t.Length; i++)
{
var s = files.Dequeue();
try
{
DBReader reader = new DBReader();
DBEntry entry = reader.Read(s.Value, s.Key);
if (entry != null)
{
var current = Entries.FirstOrDefault(x => x.FileName == entry.FileName && x.Build == entry.Build);
if (current != null)
{
Entries.Remove(current);
}
Entries.Add(entry);
if (!string.IsNullOrWhiteSpace(reader.ErrorMessage))
{
_errors.Add(FormatError(s.Key, ErrorType.Warning, reader.ErrorMessage));
}
}
}
catch (ConstraintException ex) { _errors.Add(FormatError(s.Key, ErrorType.Error, "Id column contains duplicates.")); }
catch (Exception ex) { _errors.Add(FormatError(s.Key, ErrorType.Error, ex.Message)); }
if (i % 100 == 0 && i > 0)
{
ForceGC();
}
}
ForceGC();
});
batchBlock.LinkTo(actionBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
foreach (KeyValuePair <string, MemoryStream> i in streams)
{
await batchBlock.SendAsync(i); // wait synchronously for the block to accept.
}
batchBlock.Complete();
await actionBlock.Completion;
ForceGC();
return(_errors);
}
// Adds random employee data to the database by using dataflow.
// This method is similar to AddEmployees except that it uses batching
// to add multiple employees to the database at a time.
static void AddEmployeesBatched(string connectionString, int batchSize, int count)
{
// Create a BatchBlock<Employee> that holds several Employee objects and
// then propagates them out as an array.
BatchBlock <Employee> batchEmployees = new BatchBlock <Employee>(batchSize);
// Create an ActionBlock<Employee[]> object that adds multiple
// employee entries to the database.
ActionBlock <Employee[]> insertEmployees = new ActionBlock <Employee[]>(a =>
{
DatabaseUtilities.InsertEmployees(a, connectionString, "AddEmployeesBatched");
});
// Link the batch block to the action block.
batchEmployees.LinkTo(insertEmployees);
// When the batch block completes, set the action block also to complete.
batchEmployees.Completion.ContinueWith(obj =>
{
insertEmployees.Complete();
});
// Post several random Employee objects to the batch block.
PostRandomEmployees(batchEmployees, count);
// Set the batch block to the completed state and wait for
// all insert operations to complete.
batchEmployees.Complete();
insertEmployees.Completion.Wait();
}
private void BlockComplete()
{
_source.Complete();
_batchBlock.Complete();
_transformBlock.Complete();
_writer.Complete();
}
public SqlServerBackend(string connectionString,
string collectorName,
ISystemMetricsService systemMetrics,
int retries = 3,
int batchSize = 50)
{
_log = SuperCheapIOC.Resolve <ILog>();
_connectionString = connectionString;
_collectorName = collectorName;
_systemMetrics = systemMetrics;
_retries = retries;
InitialiseRetryHandling();
_batchBlock = new BatchBlock <GraphiteLine>(batchSize);
_actionBlock = new ActionBlock <GraphiteLine[]>(p => SendToDB(p), new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 1
});
_batchBlock.LinkTo(_actionBlock);
_batchBlock.Completion.ContinueWith(p => _actionBlock.Complete());
_actionBlock.Completion.ContinueWith(p => { _isActive = false; });
_completionTask = new Task(() =>
{
_log.Info("SqlServerBackend - Completion has been signaled. Waiting for action block to complete.");
_batchBlock.Complete();
_actionBlock.Completion.Wait();
});
}
public void Start()
{
const int numberChunks = 30;
watch = new Stopwatch();
watch.Start();
for (int j = 1; j <= numberChunks; j++)
{
int collectionSize = 1000 * j;
List <InputObject> collection = new List <InputObject>(collectionSize);
for (int i = 0; i < collectionSize; i++)
{
collection.Add(new InputObject(i));
}
tempBCB.Post(collection);
}
tempBCB.Complete();
Task.WhenAll(core1.transformBlock.Completion, core2.transformBlock.Completion).ContinueWith(_ =>
{
batchBlock.Complete();
});
transformBlock.Completion.Wait();
Console.WriteLine("Completed");
Console.ReadLine();
}
public async Task BatchBlockWillCompleteTarget()
{
BatchBlock <int> bb = new BatchBlock <int>(batchSize: 2,
dataflowBlockOptions: new GroupingDataflowBlockOptions()
{
BoundedCapacity = 3
});
TestTargetBlock <int[]> testTarget = new TestTargetBlock <int[]>();
testTarget.ConsumptionMode = DataflowMessageStatus.Accepted;
bb.LinkTo(testTarget, PropagateCompletion);
// Rapidly send 50 messages
Task.WaitAll(Enumerable.Range(0, 50).Select((i) => bb.SendAsync(i)).ToArray(), BurstTimeout);
bb.Complete();
// Completion should run to successful conclusion
await Task.WhenAny(bb.Completion, Task.Delay(CompletionTimeout));
Assert.Equal(TaskStatus.RanToCompletion, bb.Completion.Status);
// Assumption: BufferBlock should also have completed its target
await Task.WhenAny(testTarget.Completion, Task.Delay(CompletionTimeout));
Assert.Equal(TaskStatus.RanToCompletion, testTarget.Completion.Status);
// Assumption: we should have gotten 25 batches
bool allMessagesReceived = await TaskUtils.PollWaitAsync(() => testTarget.MessagesConsumed.Count == 25, MessageArrivalTimeout);
Assert.True(allMessagesReceived);
}
/// <summary>
/// GeneratePipeline_DataFlowSource_to_DataFlowDestination generates a TPL-DataFlowPipeline between two vertices of a graph.
/// v_source.UserDefinedObjects[0] has to be Type of IDataFlowSource
/// v_dest.UserDefinedObjects[0] has to be Type of IDataFlowDestination
/// </summary>
/// <param name="v_source"></param>
/// <param name="v_dest"></param>
/// <param name="ToCompleteCollection"></param>
/// <param name="WatingForCompletitionCollection"></param>
private void GeneratePipeline_DataFlowSource_to_DataFlowDestination(Vertex v_source, Vertex v_dest, ref List <object> ToCompleteCollection, ref List <object> WatingForCompletitionCollection, ref Dictionary <IDataFlowSource <DS>, object> DataFlowReaderCollection)
{
IDataFlowSource <DS> t_b_source = (IDataFlowSource <DS>)v_source.UserDefinedObjects[0];
IDataFlowDestination <DS> dest = (IDataFlowDestination <DS>)v_dest.UserDefinedObjects[0];
TransformBlock <DS, DS> t_b_dummy = new TransformBlock <DS, DS>(DS => { return(DS); }
, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = this.MaxDegreeOfParallelism
});
ToCompleteCollection.Add(t_b_dummy);
v_dest.UserDefinedObjects.Add(t_b_dummy);
DataFlowReaderCollection.Add(t_b_source, t_b_dummy);
var bacthBlock = new BatchBlock <DS>(BatchSize);
var DataFlowDestinationBlock = new ActionBlock <DS[]>(outp => dest.WriteBatch(outp));
t_b_dummy.LinkTo(bacthBlock, linkOptions);
bacthBlock.LinkTo(DataFlowDestinationBlock, linkOptions);
t_b_dummy.Completion.ContinueWith(t => { bacthBlock.Complete(); });
bacthBlock.Completion.ContinueWith(t => { DataFlowDestinationBlock.Complete(); });
WatingForCompletitionCollection.Add(DataFlowDestinationBlock);
}
static void ShowBatchBlock()
{
// <snippet7>
// Create a BatchBlock<int> object that holds ten
// elements per batch.
var batchBlock = new BatchBlock <int>(10);
// Post several values to the block.
for (int i = 0; i < 13; i++)
{
batchBlock.Post(i);
}
// Set the block to the completed state. This causes
// the block to propagate out any remaining
// values as a final batch.
batchBlock.Complete();
// Print the sum of both batches.
Console.WriteLine("The sum of the elements in batch 1 is {0}.",
batchBlock.Receive().Sum());
Console.WriteLine("The sum of the elements in batch 2 is {0}.",
batchBlock.Receive().Sum());
/* Output:
* The sum of the elements in batch 1 is 45.
* The sum of the elements in batch 2 is 33.
*/
// </snippet7>
}
private static async Task Synch(CancellationToken cancellationToken)
{
BatchBlock <int> syncBatch = new BatchBlock <int>(1);
ActionBlock <int[]> syncAction = new ActionBlock <int[]>(async i =>
{
Console.WriteLine("syncAction");
//exists pairs
throw null;
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 2
});
_ = syncBatch.LinkTo(syncAction, new DataflowLinkOptions {
PropagateCompletion = true
});
await foreach (int i in AsyncEnumerable(cancellationToken))
{
bool isMsgRecived = await syncBatch.SendAsync(i);
if (isMsgRecived is false || syncAction.Completion.Status == TaskStatus.Faulted)
{
break;
}
}
syncBatch.Complete();
await syncAction.Completion;
}
public async Task TestPrecancellation()
{
var b = new BatchBlock <int>(42, new GroupingDataflowBlockOptions
{
CancellationToken = new CancellationToken(canceled: true),
MaxNumberOfGroups = 1
});
Assert.Equal(expected: 42, actual: b.BatchSize);
Assert.NotNull(b.LinkTo(DataflowBlock.NullTarget <int[]>()));
Assert.False(b.Post(42));
Task <bool> t = b.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
int[] ignoredValue;
IList <int[]> ignoredValues;
Assert.False(b.TryReceive(out ignoredValue));
Assert.False(b.TryReceiveAll(out ignoredValues));
Assert.Equal(expected: 0, actual: b.OutputCount);
Assert.NotNull(b.Completion);
b.Complete(); // verify doesn't throw
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => b.Completion);
}
public static async Task <List <string> > LoadFiles(IEnumerable <string> filenames)
{
ConcurrentBag <string> _errors = new ConcurrentBag <string>();
ConcurrentQueue <string> files = new ConcurrentQueue <string>(filenames.Distinct().OrderBy(x => x).ThenByDescending(x => Path.GetExtension(x)));
string firstFile = files.First();
var batchBlock = new BatchBlock <string>(100, new GroupingDataflowBlockOptions {
BoundedCapacity = 100
});
var actionBlock = new ActionBlock <string[]>(t =>
{
for (int i = 0; i < t.Length; i++)
{
string file;
files.TryDequeue(out file);
try
{
DBReader reader = new DBReader();
DBEntry entry = reader.Read(file);
if (entry != null)
{
var current = Entries.FirstOrDefault(x => x.FileName == entry.FileName && x.Build == entry.Build);
if (current != null)
{
Entries.Remove(current);
}
Entries.Add(entry);
if (file != firstFile)
{
entry.Detach();
}
if (!string.IsNullOrWhiteSpace(reader.ErrorMessage))
{
_errors.Add(FormatError(file, ErrorType.Warning, reader.ErrorMessage));
}
}
}
catch (ConstraintException ex) { _errors.Add(FormatError(file, ErrorType.Error, "Id column contains duplicates.")); }
catch (Exception ex) { _errors.Add(FormatError(file, ErrorType.Error, ex.Message)); }
}
ForceGC();
});
batchBlock.LinkTo(actionBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
foreach (string i in files)
{
await batchBlock.SendAsync(i); // wait synchronously for the block to accept.
}
batchBlock.Complete();
await actionBlock.Completion;
files = null;
return(_errors.ToList());
}
static void Main(string[] args)
{
var batchBlock = new BatchBlock <int>(3);
for (int i = 0; i < 10; i++)
{
batchBlock.Post(i);
}
batchBlock.Complete();
batchBlock.Post(10);
for (int i = 0; i < 5; i++)
{
if (batchBlock.TryReceive(out var result))
{
Console.Write($"Received batch {i}: ");
foreach (var r in result)
{
Console.Write(r + " ");
}
Console.WriteLine();
}
else
{
Console.WriteLine("block finished");
break;
}
}
Console.WriteLine("done");
Console.ReadKey();
}
public async Task TestNonGreedy()
{
var batch = new BatchBlock <int>(2, new GroupingDataflowBlockOptions {
Greedy = false
});
for (int trial = 0; trial < 2; trial++)
{
Task <bool> firstSend = batch.SendAsync(1 + trial);
Assert.False(firstSend.IsCompleted); // should always pass, but due to race might not test what we really want it to
Assert.Equal(expected: 0, actual: batch.OutputCount); // ditto
Task <bool> secondSend = batch.SendAsync(3 + trial);
Assert.Equal(expected: true, actual: await firstSend);
Assert.Equal(expected: true, actual: await secondSend);
Assert.Equal(expected: true, actual: await batch.OutputAvailableAsync());
Assert.Equal(expected: 1, actual: batch.OutputCount);
int[] result = await batch.ReceiveAsync();
Assert.NotNull(result);
Assert.Equal(expected: 2, actual: result.Length);
Assert.Equal(expected: 1 + trial, actual: result[0]);
Assert.Equal(expected: 3 + trial, actual: result[1]);
}
batch.Complete();
await batch.Completion;
}
public void TestTriggerBatch_Nop()
{
const int Iters = 2;
var b = new BatchBlock <int>(1);
for (int i = 0; i < Iters; i++)
{
b.Post(i);
int outputCount = b.OutputCount;
Assert.Equal(expected: i + 1, actual: outputCount);
b.TriggerBatch();
Assert.Equal(expected: outputCount, actual: b.OutputCount);
}
b = new BatchBlock <int>(1);
Assert.Equal(expected: 0, actual: b.OutputCount);
for (int i = 0; i < 2; i++)
{
b.TriggerBatch();
}
for (int i = 0; i < 2; i++)
{
b.Complete();
b.TriggerBatch();
}
Assert.Equal(expected: 0, actual: b.OutputCount);
}
public Task DownloadThemAllAsync(IEnumerable <Uri> uris, ProcessResult processResult, byte maxThreads)
{
//Multiple Consumers
var consumers = new ActionBlock <Uri[]>( //Handlers
uriBatch => ConsumerAsync(uriBatch, processResult), //!SPOT: thread-safety should be guaranteed by us
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxThreads
});
var sharedUris = new BatchBlock <Uri>(2 * maxThreads); //Command
//This is is the link between the pipeline's command and handlers
//!SPOT: Link together producer to consumers
sharedUris.LinkTo(consumers,
new DataflowLinkOptions {
PropagateCompletion = true
}); //!SPOT: without this option the consumers will never finish
//Single Producer
Task.Run(async() => {
await ProducerAsync(sharedUris, uris);
//Signaling producing is over
sharedUris.Complete();
});
//Waiting for all consumers to finish
return(consumers.Completion);
}
public async Task TestLinkToOptions()
{
const int Messages = 2;
foreach (bool append in DataflowTestHelpers.BooleanValues)
{
var bb = new BatchBlock <int>(1);
var values = new int[Messages][];
var targets = new ActionBlock <int[]> [Messages];
for (int i = 0; i < Messages; i++)
{
int slot = i;
targets[i] = new ActionBlock <int[]>(item => values[slot] = item);
bb.LinkTo(targets[i], new DataflowLinkOptions {
MaxMessages = 1, Append = append
});
}
bb.PostRange(0, Messages);
bb.Complete();
await bb.Completion;
for (int i = 0; i < Messages; i++)
{
targets[i].Complete();
await targets[i].Completion;
Assert.Equal(
expected: append ? i : Messages - i - 1,
actual: values[i][0]);
}
}
}
public async Task BufferBlocksToBatchNonGreedyToAction()
{
var inputs = Enumerable.Range(0, 1).Select(_ => new BufferBlock <int>()).ToList();
var b = new BatchBlock <int>(inputs.Count);
int completedCount = 0;
var c = new ActionBlock <int[]>(i => completedCount++);
foreach (var input in inputs)
{
input.LinkTo(b);
}
var ignored = Task.WhenAll(inputs.Select(s => s.Completion)).ContinueWith(
_ => b.Complete(), CancellationToken.None, TaskContinuationOptions.None, TaskScheduler.Default);
b.LinkTo(c, new DataflowLinkOptions {
PropagateCompletion = true
});
for (int i = 0; i < Iterations; i++)
{
inputs[i % inputs.Count].Post(i);
}
foreach (var input in inputs)
{
input.Complete();
}
await c.Completion;
Assert.Equal(expected: Iterations / b.BatchSize, actual: completedCount);
}
private void ConfigureEntity <T>(Func <Message, T> action, Action <T[]> execution, int batchSize, DataflowLinkOptions linkOptions)
where T : class
{
var transformBlock = new TransformBlock <Message, T>(action);
var batchBlock = new BatchBlock <T>(batchSize);
var actionBlock = new ActionBlock <T[]>(m =>
{
var temp = m.Where(x => x != null).ToArray();
var entity = temp.GetType().Name;
_logger.LogDebug($"Bulk insert {entity} - {temp.Length}");
execution.Invoke(temp);
_logger.LogInformation($"Process finished for {entity}");
});
batchBlock.LinkTo(actionBlock, linkOptions);
batchBlock.Completion.ContinueWith(delegate { actionBlock.Complete(); });
transformBlock.LinkTo(batchBlock, linkOptions);
transformBlock.Completion.ContinueWith(delegate { batchBlock.Complete(); });
var bufferBlock = new BufferBlock <Message>();
bufferBlock.LinkTo(transformBlock, linkOptions, m => m != null && m.Type == typeof(T).Name);
_bufferBlocks.Add(typeof(T).Name, bufferBlock);
}
/// <summary>
/// The following basic example posts several Int32 values to a BatchBlock<T> object
/// that holds ten elements in a batch. To guarantee that all values propagate out of the
/// BatchBlock<T>, this example calls the Complete method. The Complete method sets the
/// BatchBlock<T> object to the completed state, and therefore, the BatchBlock<T> object
/// propagates out any remaining elements as a final batch.
/// </summary>
public static void Run()
{
// Create a BatchBlock<int> object that holds ten
// elements per batch.
// Provides a dataflow block that batches inputs into arrays
// Below every batch will have 10 items.
BatchBlock <int> batchBlock = new BatchBlock <int>(10);
Console.WriteLine("BatchBlock<int> batchBlock = new BatchBlock<int>(10);");
// Post several values to the block.
for (int i = 0; i < 13; i++)
{
batchBlock.Post(i);
Console.WriteLine("Posted to batchBlock {0}.", i);
}
// Set the block to the completed state. This causes
// the block to propagate out any any remaining
// values as a final batch.
batchBlock.Complete();
Console.WriteLine("batchBlock.Complete();");
// Print the sum of both batches.
int length1 = batchBlock.Receive().Length;
int length2 = batchBlock.Receive().Length;
Console.WriteLine("The total element count in batch-1 is {0}.", length1);
Console.WriteLine("The total element count in batch-2 is {0}.", length2);
}
public static void Main(string[] args)
{
var participants = Participant.Generate(100000, 100000, CompanyId).ToArray();
var actionBatchBlock = new BatchBlock <ActionParticipant>(7500);
var insertActionParticipants = new ActionBlock <ActionParticipant[]>(items =>
{
var bulkInsertAsync = DataImporter.BulkInsertAsync(items);
Inserted += items.Length;
Console.WriteLine($"Inserted {Inserted} out of { ActionCount}");
return(bulkInsertAsync);
});
actionBatchBlock.LinkTo(insertActionParticipants);
actionBatchBlock.Completion.ContinueWith(delegate { insertActionParticipants.Complete(); });
foreach (var actionParticipant in ActionParticipant.Generate(participants, ActionCount, 30000, CompanyId))
{
actionBatchBlock.Post(actionParticipant);
}
actionBatchBlock.Complete();
insertActionParticipants.Completion.Wait();
}
public void Run()
{
var batchBlock = new BatchBlock <int>(10);
for (int i = 0; i < 20; i++)
{
batchBlock.Post(i);
}
batchBlock.Complete();
batchBlock.Post(10);
for (int i = 0; i < 5; i++)
{
int[] result;
if (batchBlock.TryReceive(out result))
{
Console.Write($"Received batch {i}:");
foreach (var r in result)
{
Console.Write(r + " ");
}
Console.Write("\n");
}
else
{
Console.WriteLine("The block finished");
break;
}
}
}
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics)
{
_systemMetrics = systemMetrics;
var config = new SqlServerConfiguration(configElement.Attribute("connectionString").Value,
configElement.ToInt("writeBatchSize"));
_connectionString = config.ConnectionString;
_collectorName = collectorName;
_retries = config.Retries;
InitialiseRetryHandling();
_batchBlock = new BatchBlock <GraphiteLine>(config.WriteBatchSize);
_actionBlock = new ActionBlock <GraphiteLine[]>(p => SendToDB(p),
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1
});
_batchBlock.LinkTo(_actionBlock);
_batchBlock.Completion.ContinueWith(p => _actionBlock.Complete());
_actionBlock.Completion.ContinueWith(p => { IsActive = false; });
Completion = new Task(() =>
{
_log.Info("SqlServerBackend - Completion has been signaled. Waiting for action block to complete.");
_batchBlock.Complete();
_actionBlock.Completion.Wait();
});
}
// A BatchBlock takes in multiple types of values and outputs those values joined together (amount specified in the constructor)
private static async Task SimpleDemoAsync()
{
Console.WriteLine("BatchBlockDemo has started!");
var block = new BatchBlock <int>(3); // batch messages of type int, into 3
for (int i = 0; i < 10; i++)
{
block.Post(i);
}
block.Complete(); // No mo data.
while (await block.OutputAvailableAsync().ConfigureAwait(false))
{
var output = await block.ReceiveAsync().ConfigureAwait(false);
Console.WriteLine($"BatchBlock BatchOutput: {string.Join(",", output)}");
Console.WriteLine($"BatchBlock OutputCount: {block.OutputCount}");
}
// wait for completion.
await block.Completion.ConfigureAwait(false);
Console.WriteLine("Finished!");
Console.ReadKey();
}
static async Task<long> skynetTpl( long num, long size, long div )
{
BatchBlock<long> source = new BatchBlock<long>( 1024 );
long sum = 0;
ActionBlock<long[]> actAggregate = new ActionBlock<long[]>( vals => sum += vals.Sum(),
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1, SingleProducerConstrained = true } );
source.LinkTo( actAggregate, new DataflowLinkOptions() { PropagateCompletion = true } );
skynetTplRecursion( source, num, size, div );
source.Complete();
await actAggregate.Completion;
return sum;
}
public async Task TestNonGreedyLostMessages()
{
foreach (int batchSize in new[] { 2, 5 })
{
var batch = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions { Greedy = false });
var buffers = Enumerable.Range(0, batchSize - 1).Select(_ => new BufferBlock<int>()).ToList();
var tcs = new TaskCompletionSource<bool>();
int remaining = buffers.Count;
// Offer the batch almost all the messages it needs, but have them consumed by someone else
foreach (var buffer in buffers)
{
buffer.LinkTo(batch);
buffer.LinkTo(new ActionBlock<int>(i => {
if (Interlocked.Decrement(ref remaining) == 0)
{
tcs.SetResult(true);
}
}));
buffer.Post(42);
}
await tcs.Task;
// Now offer from another set of sources that won't lose them
buffers = Enumerable.Range(0, batchSize).Select(_ => new BufferBlock<int>()).ToList();
foreach (var buffer in buffers)
{
buffer.LinkTo(batch);
buffer.Post(43);
buffer.Complete();
}
// Wait until all the messages are consumed
await Task.WhenAll(from buffer in buffers select buffer.Completion);
int[] results = await batch.ReceiveAsync();
Assert.Equal(expected: 0, actual: batch.OutputCount);
batch.Complete();
await batch.Completion;
}
}
public async Task BatchGreedyToAction()
{
var b = new BatchBlock<int>(1);
int completedCount = 0;
var c = new ActionBlock<int[]>(i => completedCount++);
b.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true });
b.PostRange(0, Iterations);
b.Complete();
await c.Completion;
Assert.Equal(expected: Iterations / b.BatchSize, actual: completedCount);
}
public async Task TestProducerConsumer()
{
foreach (TaskScheduler scheduler in new[] { TaskScheduler.Default, new ConcurrentExclusiveSchedulerPair().ExclusiveScheduler })
foreach (int maxMessagesPerTask in new[] { DataflowBlockOptions.Unbounded, 1, 2 })
foreach (int boundedCapacity in new[] { DataflowBlockOptions.Unbounded, 2, 3 })
foreach (int batchSize in new[] { 1, 2 })
{
const int Messages = 50;
var bb = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions
{
BoundedCapacity = boundedCapacity,
MaxMessagesPerTask = maxMessagesPerTask,
TaskScheduler = scheduler
});
await Task.WhenAll(
Task.Run(async delegate { // consumer
int i = 0;
while (await bb.OutputAvailableAsync())
{
int[] items = await bb.ReceiveAsync();
Assert.Equal(expected: batchSize, actual: items.Length);
for (int j = 0; j < items.Length; j++)
{
Assert.Equal(expected: i + j, actual: items[j]);
}
i += batchSize;
}
}),
Task.Run(async delegate { // producer
for (int i = 0; i < Messages; i++)
{
await bb.SendAsync(i);
}
bb.Complete();
}));
}
}
public async Task TestLinkToOptions()
{
const int Messages = 2;
foreach (bool append in DataflowTestHelpers.BooleanValues)
{
var bb = new BatchBlock<int>(1);
var values = new int[Messages][];
var targets = new ActionBlock<int[]>[Messages];
for (int i = 0; i < Messages; i++)
{
int slot = i;
targets[i] = new ActionBlock<int[]>(item => values[slot] = item);
bb.LinkTo(targets[i], new DataflowLinkOptions { MaxMessages = 1, Append = append });
}
bb.PostRange(0, Messages);
bb.Complete();
await bb.Completion;
for (int i = 0; i < Messages; i++)
{
targets[i].Complete();
await targets[i].Completion;
Assert.Equal(
expected: append ? i : Messages - i - 1,
actual: values[i][0]);
}
}
}
public async Task BufferBlocksToBatchNonGreedyToAction()
{
var inputs = Enumerable.Range(0, 1).Select(_ => new BufferBlock<int>()).ToList();
var b = new BatchBlock<int>(inputs.Count);
int completedCount = 0;
var c = new ActionBlock<int[]>(i => completedCount++);
foreach (var input in inputs) input.LinkTo(b);
var ignored = Task.WhenAll(inputs.Select(s => s.Completion)).ContinueWith(
_ => b.Complete(), CancellationToken.None, TaskContinuationOptions.None, TaskScheduler.Default);
b.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true });
for (int i = 0; i < Iterations; i++)
{
inputs[i % inputs.Count].Post(i);
}
foreach (var input in inputs) input.Complete();
await c.Completion;
Assert.Equal(expected: Iterations / b.BatchSize, actual: completedCount);
}
public async Task TestBoundedReceives()
{
for (int test = 0; test < 4; test++)
{
var bb = new BatchBlock<int>(1, new GroupingDataflowBlockOptions { BoundedCapacity = 1 });
Assert.True(bb.Post(0));
int[] item;
const int sends = 5;
for (int i = 1; i <= sends; i++)
{
Task<bool> send = bb.SendAsync(i);
Assert.True(await bb.OutputAvailableAsync()); // wait for previously posted/sent item
switch (test)
{
case 0:
IList<int[]> items;
Assert.True(bb.TryReceiveAll(out items));
Assert.Equal(expected: 1, actual: items.Count);
Assert.Equal(expected: 1, actual: items[0].Length);
Assert.Equal(expected: i - 1, actual: items[0][0]);
break;
case 1:
Assert.True(bb.TryReceive(f => true, out item));
Assert.Equal(expected: 1, actual: item.Length);
Assert.Equal(expected: i - 1, actual: item[0]);
break;
case 2:
Assert.False(bb.TryReceive(f => f.Length == 1 && f[0] == i, out item));
Assert.True(bb.TryReceive(f => f.Length == 1 && f[0] == i - 1, out item));
Assert.Equal(expected: 1, actual: item.Length);
Assert.Equal(expected: i - 1, actual: item[0]);
break;
case 3:
item = await bb.ReceiveAsync();
Assert.Equal(expected: 1, actual: item.Length);
Assert.Equal(expected: i - 1, actual: item[0]);
break;
}
}
// Receive remaining item
item = await bb.ReceiveAsync();
Assert.Equal(expected: 1, actual: item.Length);
Assert.Equal(expected: sends, actual: item[0]);
bb.Complete();
await bb.Completion;
}
}
public async Task TestCompletionWithBufferedItems()
{
var b = new BatchBlock<int>(5);
b.PostRange(0, 3);
b.Complete();
await b.OutputAvailableAsync();
Assert.Equal(expected: 1, actual: b.OutputCount);
int[] items = await b.ReceiveAsync();
Assert.Equal(expected: 3, actual: items.Length);
await b.Completion;
}
public async Task TestNonGreedy()
{
var batch = new BatchBlock<int>(2, new GroupingDataflowBlockOptions { Greedy = false });
for (int trial = 0; trial < 2; trial++)
{
Task<bool> firstSend = batch.SendAsync(1 + trial);
Assert.False(firstSend.IsCompleted); // should always pass, but due to race might not test what we really want it to
Assert.Equal(expected: 0, actual: batch.OutputCount); // ditto
Task<bool> secondSend = batch.SendAsync(3 + trial);
Assert.Equal(expected: true, actual: await firstSend);
Assert.Equal(expected: true, actual: await secondSend);
Assert.Equal(expected: true, actual: await batch.OutputAvailableAsync());
Assert.Equal(expected: 1, actual: batch.OutputCount);
int[] result = await batch.ReceiveAsync();
Assert.NotNull(result);
Assert.Equal(expected: 2, actual: result.Length);
Assert.Equal(expected: 1 + trial, actual: result[0]);
Assert.Equal(expected: 3 + trial, actual: result[1]);
}
batch.Complete();
await batch.Completion;
}
internal static bool BatchGreedyToAction()
{
const int ITERS = 2;
var b = new BatchBlock<int>(1);
int completedCount = 0;
var c = new ActionBlock<int[]>(i => completedCount++);
b.LinkWithCompletion(c);
for (int i = 0; i < ITERS; i++) b.Post(i);
b.Complete();
c.Completion.Wait();
return completedCount == ITERS / b.BatchSize;
}
public async Task TestPrecancellation()
{
var b = new BatchBlock<int>(42, new GroupingDataflowBlockOptions {
CancellationToken = new CancellationToken(canceled: true), MaxNumberOfGroups = 1
});
Assert.Equal(expected: 42, actual: b.BatchSize);
Assert.NotNull(b.LinkTo(DataflowBlock.NullTarget<int[]>()));
Assert.False(b.Post(42));
Task<bool> t = b.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
int[] ignoredValue;
IList<int[]> ignoredValues;
Assert.False(b.TryReceive(out ignoredValue));
Assert.False(b.TryReceiveAll(out ignoredValues));
Assert.Equal(expected: 0, actual: b.OutputCount);
Assert.NotNull(b.Completion);
b.Complete(); // verify doesn't throw
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => b.Completion);
}
internal static bool BufferBlocksToBatchNonGreedyToAction()
{
const int ITERS = 2;
var inputs = Enumerable.Range(0, 1).Select(_ => new BufferBlock<int>()).ToList();
var b = new BatchBlock<int>(inputs.Count);
int completedCount = 0;
var c = new ActionBlock<int[]>(i => completedCount++);
foreach (var input in inputs) input.LinkTo(b);
Task.Factory.ContinueWhenAll(inputs.Select(i => i.Completion).ToArray(), _ => b.Complete());
b.LinkWithCompletion(c);
for (int i = 0; i < ITERS; i++)
{
inputs[i % inputs.Count].Post(i);
}
foreach (var input in inputs) input.Complete();
c.Completion.Wait();
return (completedCount == ITERS / b.BatchSize);
}
public void TestTriggerBatch_Nop()
{
const int Iters = 2;
var b = new BatchBlock<int>(1);
for (int i = 0; i < Iters; i++)
{
b.Post(i);
int outputCount = b.OutputCount;
Assert.Equal(expected: i + 1, actual: outputCount);
b.TriggerBatch();
Assert.Equal(expected: outputCount, actual: b.OutputCount);
}
b = new BatchBlock<int>(1);
Assert.Equal(expected: 0, actual: b.OutputCount);
for (int i = 0; i < 2; i++)
{
b.TriggerBatch();
}
for (int i = 0; i < 2; i++)
{
b.Complete();
b.TriggerBatch();
}
Assert.Equal(expected: 0, actual: b.OutputCount);
}
public void Init()
{
_massMail = new MassMail(Config.BlockSize, Config.UserAgent, Config.ConnectionString, Config.Mode);
_templateCache = new ConcurrentDictionary<long, Lazy<Template>>();
_attachmentCache = new ConcurrentDictionary<long, Lazy<Attach>>();
_dkimSignerCache = new ConcurrentDictionary<string, DkimSigner>();
_domailKeySignerCache = new ConcurrentDictionary<string, DomainKeySigner>();
//Get all private keys
GetDkimSigners();
//*** Create pipeline ***
//Create TransformBlock that gets table of client data and make a list of objects from them.
_parseXmlDataBlock = new TransformBlock<DataTable, List<Mail>>(sendData => ParseXmlData(sendData),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Config.ParseXmlMaxdop,
BoundedCapacity = Config.ParseXmlBufferSize
});
//Create TransformBlock that gets a list of client objects, send them email, and stores result in DataTable.
_sendEmailsBlock = new TransformBlock<List<Mail>, DataTable>(mails => SendEmails(_massMail, mails),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Config.SendEmailsMaxdop,
BoundedCapacity = Config.SendEmailsMaxdop
});
//Create BatchBlock that holds several DataTable and then propagates them out as an array.
_batchResultBlock = new BatchBlock<DataTable>(Config.BatchSize,
new GroupingDataflowBlockOptions
{
BoundedCapacity = Config.BatchSize
});
//Create ActionBlock that writes result into DB
_writeResultsBlock = new ActionBlock<DataTable[]>(results => WriteResults(_massMail, results),
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1
});
//*** Build pipeline ***
// POST --> _parseXmlDataBlock --> _sendEmailsBlock --> _batchResultBlock --> _writeResultsBlock
_parseXmlDataBlock.LinkTo(_sendEmailsBlock);
_sendEmailsBlock.LinkTo(_batchResultBlock);
_batchResultBlock.LinkTo(_writeResultsBlock);
_parseXmlDataBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)_sendEmailsBlock).Fault(t.Exception);
else _sendEmailsBlock.Complete();
});
_sendEmailsBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)_batchResultBlock).Fault(t.Exception);
else _batchResultBlock.Complete();
});
_batchResultBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)_writeResultsBlock).Fault(t.Exception);
else _writeResultsBlock.Complete();
});
}
private static bool TestTriggerBatch(int boundedCapacity)
{
bool passed = true;
// Test greedy with batch size of 1 (force should always be a nop)
{
bool localPassed = true;
const int ITERS = 2;
var b = new BatchBlock<int>(1, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity });
for (int i = 0; i < ITERS; i++)
{
b.Post(i);
int outputCount = b.OutputCount;
b.TriggerBatch();
localPassed &= outputCount == b.OutputCount;
}
localPassed &= b.OutputCount == ITERS;
for (int i = 0; i < ITERS; i++)
{
var arr = b.Receive();
localPassed &= arr.Length == 1 && arr[0] == i;
}
Assert.True(localPassed, string.Format("greedy with batch size of 1 - {0}", localPassed ? "Passed" : "FAILED"));
}
// Test greedy with varying batch sizes and smaller queued numbers
{
bool localPassed = true;
foreach (var batchSize in new[] { 3 })
{
foreach (var queuedBeforeTrigger in new[] { 1, batchSize - 1 })
{
var b = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity });
for (int p = 1; p <= queuedBeforeTrigger; p++) b.Post(p);
localPassed &= b.OutputCount == 0;
b.TriggerBatch();
b.OutputAvailableAsync().Wait(); // The previous batch is triggered asynchronously when non-Unbounded BoundedCapacity is provided
localPassed &= b.OutputCount == 1 && b.Receive().Length == queuedBeforeTrigger;
for (int j = 0; j < batchSize; j++)
{
localPassed &= b.OutputCount == 0;
b.Post(j);
}
localPassed &= b.OutputCount == 1;
}
}
Assert.True(localPassed, string.Format("greedy with varying batch sizes - {0}", localPassed ? "Passed" : "FAILED"));
passed &= localPassed;
}
// Test greedy with empty queue
{
bool localPassed = true;
foreach (var batchSize in new[] { 1 })
{
var b = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity });
for (int i = 0; i < 2; i++) b.TriggerBatch();
localPassed &= b.OutputCount == 0;
}
Assert.True(localPassed, string.Format("greedy with empty queue - {0}", localPassed ? "Passed" : "FAILED"));
passed &= localPassed;
}
// Test greedy after decline
{
bool localPassed = true;
{
var b = new BatchBlock<int>(2, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity });
localPassed &= b.OutputCount == 0;
b.Complete();
localPassed &= b.OutputCount == 0;
b.TriggerBatch();
localPassed &= b.OutputCount == 0;
}
{
var b = new BatchBlock<int>(2, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity });
localPassed &= b.OutputCount == 0;
b.Post(1);
localPassed &= b.OutputCount == 0;
b.Complete();
localPassed &= b.OutputCount == 1;
b.TriggerBatch();
localPassed &= b.OutputCount == 1;
}
Assert.True(localPassed, string.Format("greedy after decline - {0}", localPassed ? "Passed" : "FAILED"));
passed &= localPassed;
}
// Test greedy after canceled
{
bool localPassed = true;
{
var cts = new CancellationTokenSource();
var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, BoundedCapacity = boundedCapacity };
var b = new BatchBlock<int>(2, dbo);
localPassed &= b.OutputCount == 0;
cts.Cancel();
localPassed &= b.OutputCount == 0;
b.TriggerBatch();
localPassed &= b.OutputCount == 0;
}
{
var cts = new CancellationTokenSource();
var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, BoundedCapacity = boundedCapacity };
var b = new BatchBlock<int>(2, dbo);
localPassed &= b.OutputCount == 0;
b.Post(1);
localPassed &= b.OutputCount == 0;
cts.Cancel();
localPassed &= b.OutputCount == 0;
b.TriggerBatch();
localPassed &= b.OutputCount == 0;
}
Assert.True(localPassed, string.Format("greedy after canceled - {0}", localPassed ? "Passed" : "FAILED"));
passed &= localPassed;
}
// Test greedy with MaxNumberOfGroups == 1
{
bool localPassed = true;
var b = new BatchBlock<int>(2, new GroupingDataflowBlockOptions { MaxNumberOfGroups = 1, BoundedCapacity = boundedCapacity });
b.Post(1);
localPassed &= b.OutputCount == 0;
b.TriggerBatch();
b.OutputAvailableAsync().Wait(); // The previous batch is triggered asynchronously when non-Unbounded BoundedCapacity is provided
localPassed &= b.OutputCount == 1;
localPassed &= b.Post(2) == false;
b.TriggerBatch();
localPassed &= b.OutputCount == 1;
Assert.True(localPassed, string.Format("greedy with MaxNumberOfGroups == 1 - {0}", localPassed ? "Passed" : "FAILED"));
passed &= localPassed;
}
// Test non-greedy with no queued or postponed messages
{
bool localPassed = true;
var dbo = new GroupingDataflowBlockOptions { Greedy = false, BoundedCapacity = boundedCapacity };
var b = new BatchBlock<int>(3, dbo);
localPassed &= b.OutputCount == 0;
b.TriggerBatch();
localPassed &= b.OutputCount == 0;
Assert.True(localPassed, string.Format("non-greedy with no mesages - {0}", localPassed ? "Passed" : "FAILED"));
passed &= localPassed;
}
// Test non-greedy with no queued but postponed messages
{
bool localPassed = true;
var dbo = new GroupingDataflowBlockOptions { Greedy = false, BoundedCapacity = boundedCapacity };
const int BATCH_SIZE = 10;
for (int numPostponedMessages = 1; numPostponedMessages < BATCH_SIZE; numPostponedMessages++)
{
var b = new BatchBlock<int>(BATCH_SIZE, dbo);
localPassed &= b.OutputCount == 0;
for (int i = 0; i < numPostponedMessages; i++) localPassed &= !b.SendAsync(i).IsCompleted;
b.TriggerBatch();
var output = b.Receive();
localPassed &= output.Length == numPostponedMessages;
for (int i = 0; i < output.Length; i++) localPassed &= output[i] == i;
localPassed &= b.OutputCount == 0;
b.TriggerBatch();
localPassed &= b.OutputCount == 0;
}
Assert.True(localPassed, string.Format("non-greedy with postponed, no queued - {0}", localPassed ? "Passed" : "FAILED"));
passed &= localPassed;
}
return passed;
}
public void NonGreedyCompleteDoesnTriggerBatchTest ()
{
var scheduler = new TestScheduler ();
var block = new BatchBlock<int> (2,
new GroupingDataflowBlockOptions
{ Greedy = false, TaskScheduler = scheduler });
var source =
new BufferBlock<int> (new DataflowBlockOptions { TaskScheduler = scheduler });
Assert.IsNotNull (source.LinkTo (block));
Assert.IsTrue (source.Post (1));
block.Complete ();
int[] batch;
Assert.IsFalse (block.TryReceive (out batch));
Assert.IsTrue (block.Completion.Wait (1000));
}
public void RunBatchBlockConformanceTests()
{
bool localPassed;
// Greedy batching
{
localPassed = true;
const int NUM_MESSAGES = 1;
const int BATCH_SIZE = 1;
var batch = new BatchBlock<int>(BATCH_SIZE);
for (int i = 0; i < NUM_MESSAGES * BATCH_SIZE; i++) batch.Post(i);
for (int i = 0; i < NUM_MESSAGES; i++)
{
int[] result = batch.Receive();
localPassed &= result.Length == BATCH_SIZE;
for (int j = 0; j < result.Length - 1; j++)
{
localPassed &= (result[j] + 1 == result[j + 1]);
}
}
Assert.True(localPassed, string.Format("{0}: Greedy batching", localPassed ? "Success" : "Failure"));
}
// Non-greedy batching with BATCH_SIZE sources used repeatedly
{
localPassed = true;
const int NUM_MESSAGES = 1;
const int BATCH_SIZE = 1;
var batch = new BatchBlock<int>(BATCH_SIZE, new GroupingDataflowBlockOptions { Greedy = false });
var buffers = Enumerable.Range(0, BATCH_SIZE).Select(_ => new BufferBlock<int>()).ToList();
foreach (var buffer in buffers) buffer.LinkTo(batch);
int prevSum = -1;
for (int i = 0; i < NUM_MESSAGES; i++)
{
for (int j = 0; j < BATCH_SIZE; j++) buffers[j].Post(i);
int sum = batch.Receive().Sum();
localPassed &= (sum > prevSum);
prevSum = sum;
}
Assert.True(localPassed, string.Format("{0}: Non-greedy batching with BATCH_SIZE sources used repeatedly", localPassed ? "Success" : "Failure"));
}
// Non-greedy batching with BATCH_SIZE * NUM_MESSAGES sources
{
localPassed = true;
const int NUM_MESSAGES = 1;
const int BATCH_SIZE = 2;
var batch = new BatchBlock<int>(BATCH_SIZE, new GroupingDataflowBlockOptions { Greedy = false });
var buffers = Enumerable.Range(0, BATCH_SIZE * NUM_MESSAGES).Select(_ => new BufferBlock<int>()).ToList();
foreach (var buffer in buffers)
{
buffer.LinkTo(batch);
buffer.Post(1);
}
for (int i = 0; i < NUM_MESSAGES; i++)
{
localPassed &= batch.Receive().Sum() == BATCH_SIZE;
}
Assert.True(localPassed, string.Format("{0}: Non-greedy batching with N*M sources", localPassed ? "Success" : "Failure"));
}
// Non-greedy batching with missed messages
{
localPassed = true;
const int BATCH_SIZE = 2;
var batch = new BatchBlock<int>(BATCH_SIZE, new GroupingDataflowBlockOptions { Greedy = false });
var buffers = Enumerable.Range(0, BATCH_SIZE - 1).Select(_ => new BufferBlock<int>()).ToList();
using (var ce = new CountdownEvent(BATCH_SIZE - 1))
{
foreach (var buffer in buffers)
{
buffer.LinkTo(batch);
buffer.LinkTo(new ActionBlock<int>(i => ce.Signal()));
buffer.Post(42);
}
ce.Wait();
}
buffers = Enumerable.Range(0, BATCH_SIZE).Select(_ => new BufferBlock<int>()).ToList();
foreach (var buffer in buffers)
{
buffer.LinkTo(batch);
buffer.Post(42);
buffer.Complete();
}
localPassed &= Task.WaitAll(buffers.Select(b => b.Completion).ToArray(), 2000);
Assert.True(localPassed, string.Format("{0}: Non-greedy batching with missed messages", localPassed ? "Success" : "Failure"));
}
// Test using a precanceled token
{
localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, MaxNumberOfGroups = 1 };
var b = new BatchBlock<int>(42, dbo);
int[] ignoredValue;
IList<int[]> ignoredValues;
localPassed &= b.BatchSize == 42;
localPassed &= b.LinkTo(new ActionBlock<int[]>(delegate { })) != null;
localPassed &= b.SendAsync(42).Result == false;
localPassed &= b.TryReceiveAll(out ignoredValues) == false;
localPassed &= b.Post(42) == false;
localPassed &= b.OutputCount == 0;
localPassed &= b.TryReceive(out ignoredValue) == false;
localPassed &= b.Completion != null;
b.Complete();
}
catch (Exception)
{
localPassed = false;
}
Assert.True(localPassed, string.Format("{0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"));
}
// Test completing block while still items buffered
{
localPassed = true;
var b = new BatchBlock<int>(5);
b.Post(1);
b.Post(2);
b.Post(3);
b.Complete();
localPassed &= b.Receive().Length == 3;
Assert.True(localPassed, string.Format("{0}: Makes batches of remaining items", localPassed ? "Success" : "Failure"));
}
}
private static bool TestTriggerBatchRacingWithComplete(bool greedy)
{
bool passed = true;
const int batchSize = 2;
const int iterations = 1;
const int waitTimeout = 100;
var dbo = new GroupingDataflowBlockOptions { Greedy = greedy };
for (int iter = 0; iter < iterations; iter++)
{
bool localPassed = true;
var sendAsyncTasks = new Task<bool>[batchSize - 1];
var racerReady = new ManualResetEventSlim();
int[] output1 = null;
int[] output2 = null;
// Blocks
var batch = new BatchBlock<int>(batchSize, dbo);
var terminator = new ActionBlock<int[]>(x => { if (output1 == null) output1 = x; else output2 = x; });
batch.LinkTo(terminator);
// Queue up batchSize-1 input items
for (int i = 0; i < batchSize - 1; i++) sendAsyncTasks[i] = batch.SendAsync(i);
var racer = Task.Factory.StartNew(() =>
{
racerReady.Set();
batch.Complete();
});
// Wait for the racer to get ready and trigger
localPassed &= racerReady.Wait(waitTimeout);
batch.TriggerBatch();
Assert.True(localPassed, "The racer task FAILED to start.");
if (localPassed)
{
// Wait for the SendAsync tasks to complete
localPassed &= Task.WaitAll(sendAsyncTasks, waitTimeout);
Assert.True(localPassed, "SendAsync tasks FAILED to complete");
}
// Do this verification only in greedy mode, because non-greedy is non-deterministic
if (greedy)
{
// Wait for a batch to be produced
if (localPassed)
{
localPassed &= SpinWait.SpinUntil(() => output1 != null, waitTimeout);
Assert.True(localPassed, "FAILED to produce a batch");
}
if (localPassed)
{
//Verify the number of input items propagated
localPassed &= output1.Length == batchSize - 1;
Assert.True(localPassed, string.Format("FAILED to propagate {0} input items. count1={1}",
batchSize, output1.Length));
}
}
// Wait for the block to complete
if (localPassed)
{
localPassed &= batch.Completion.Wait(waitTimeout);
Assert.True(localPassed, "The block FAILED to complete");
}
// There should never be a second batch produced
if (localPassed)
{
localPassed &= output2 == null;
Assert.True(localPassed, "FAILED not to produce a second batch");
}
passed &= localPassed;
if (!localPassed)
{
Assert.True(localPassed, string.Format("Iteration={0}", iter));
Assert.True(localPassed, string.Format("Count1={0}", output1 == null ? "null" : output1.Length.ToString()));
break;
}
}
return passed;
}
public void CompleteTriggersBatchTest ()
{
var block = new BatchBlock<int> (2);
Assert.IsTrue (block.Post (1));
block.Complete ();
CollectionAssert.AreEqual (new[] { 1 },
block.Receive (TimeSpan.FromMilliseconds (2000)));
Assert.IsTrue (block.Completion.Wait (1000));
}
private async void StartRefreshStationsAsync()
{
IsStationWorkerRunning = true;
while (refreshingPool.Count > 0)
{
await Task.Delay(15000).ConfigureAwait(false);
if (CrossConnectivity.Current.IsConnected)
{
BatchBlock<Station> batchBlock = new BatchBlock<Station>(5);
var actionBlock = new ActionBlock<Station[]>(
async stations =>
{
foreach (var station in stations)
{
if (await station.Contract.RefreshAsync(station))
{
if (station.IsUiRefreshNeeded)
{
StationRefreshed?.Invoke(station, EventArgs.Empty);
}
}
}
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 5
});
batchBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true });
foreach (var station in refreshingPool)
{
await batchBlock.SendAsync(station); // wait synchronously for the block to accept.
}
batchBlock.Complete();
actionBlock.Completion.Wait(15000);
}
}
IsStationWorkerRunning = false;
}
