/// <summary> /// 阻塞等待 /// </summary> public void Complete(bool iswait = false) { _batchBlock.Complete(); _batchBlock.Completion.Wait(); _actionBlock.Complete(); if (iswait) { _actionBlock.Completion.Wait(); } }
public IDictionary <string, string[]> BatchBlockUsage(int numberOfIteration, int batchsize) { Console.WriteLine($"Inside {nameof(TplDataflow3GroupingBlocksController)} - {nameof(BatchBlockUsage)}"); var ouputCollection = new Dictionary <string, string[]>(); Functions.ClearCounterForBatchBlockUsage(); // Create the members of the pipeline. var batchBlockWithSizeGivenInInput = new BatchBlock <string>(batchsize); var actionBlockPerformActionOnBatchData = new ActionBlock <string[]>(batchedInput => Functions.DisplayByGroups(ouputCollection, batchedInput) ); // Connect the dataflow blocks to form a pipeline. batchBlockWithSizeGivenInInput.LinkTo(actionBlockPerformActionOnBatchData, DataflowOptions.LinkOptions); // Start BatchBlockUsage pipeline with the input values. for (var i = 0; i < numberOfIteration; i++) { batchBlockWithSizeGivenInInput.Post($"Value = {i}"); } // Mark the head of the pipeline as complete. batchBlockWithSizeGivenInInput.Complete(); // Wait for the last block in the pipeline to process all messages. actionBlockPerformActionOnBatchData.Completion.Wait(); return(ouputCollection); }
public void ActionQueueTestInPipeline() { var list = new List <List <int> >(); var option = new DataflowLinkOptions { PropagateCompletion = true, }; var batchBlock = new BatchBlock <int>(10); var actionBlock = new ActionBlock <int[]>(x => list.Add(new List <int>(x))); batchBlock.LinkTo(actionBlock, option); var p = new PipelineManager <IWorkContext, int> { new Pipeline <IWorkContext, int>() + ((c, x) => { batchBlock.Post(x); return(true); }), }; Enumerable.Range(0, 100) .ForEach(x => p.Post(null !, x)); batchBlock.Complete(); Task.WaitAll(batchBlock.Completion, actionBlock.Completion); list.Count.Should().Be(10); list.All(x => x.Count == 10).Should().BeTrue(); }
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics) { _log = SuperCheapIOC.Resolve<ILog>(); _systemMetrics = systemMetrics; var config = new SqlServerConfiguration(configElement.Attribute("connectionString").Value, configElement.ToInt("writeBatchSize")); _connectionString = config.ConnectionString; _collectorName = collectorName; _retries = config.Retries; InitialiseRetryHandling(); _batchBlock = new BatchBlock<GraphiteLine>(config.WriteBatchSize); _actionBlock = new ActionBlock<GraphiteLine[]>(p => SendToDB(p), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1 }); _batchBlock.LinkTo(_actionBlock); _batchBlock.Completion.ContinueWith(p => _actionBlock.Complete()); _actionBlock.Completion.ContinueWith(p => { _isActive = false; }); _completionTask = new Task(() => { _log.Info("SqlServerBackend - Completion has been signaled. Waiting for action block to complete."); _batchBlock.Complete(); _actionBlock.Completion.Wait(); }); }
/// <summary> /// Inserts records of the appropriate type to the database /// </summary> /// <param name="broadcast"></param> /// <param name="batchSize"></param> /// <param name="context"></param> public static void Insert(BatchBlock <TEntity> broadcast, KeepaContext context) { // Create a BatchBlock<best_sellers> that holds several best_seller objects and // then propagates them out as an array. //var batchRecs = new BatchBlock<TEntity>(1000); //var queue = new BufferBlock<best_sellers>(); // Create an ActionBlock<best_seller[]> object that adds multiple // best_seller entries to the database. var insertEmployees = new ActionBlock <TEntity[]>(a => context.Set <TEntity>().AddRange(a) ); //Link broadcast to batch broadcast.LinkTo(insertEmployees, new DataflowLinkOptions { PropagateCompletion = true }); // Link the batch block to the action block. //batchRecs.LinkTo(insertEmployees, new DataflowLinkOptions { PropagateCompletion = true }); // When the batch block completes, set the action block also to complete. broadcast.Completion.ContinueWith(delegate { insertEmployees.Complete(); }); // Set the batch block to the completed state and wait for // all insert operations to complete. broadcast.Complete(); insertEmployees.Completion.Wait(); }
static void Main(string[] args) { Task.Run(async() => { bufferBlock = new BufferBlock <string>(); var batchBlock = new BatchBlock <string>(7); var action = new ActionBlock <IEnumerable <string> >(async t => { await SendUsers(t); }); batchBlock.LinkTo(action, new DataflowLinkOptions() { PropagateCompletion = true }); bufferBlock.LinkTo(batchBlock, new DataflowLinkOptions() { PropagateCompletion = true }); await InsertUsers(); Console.WriteLine("Now the buffer will complete. It's complition will be followed by completing the batch. Press any key to continue..."); Console.ReadKey(); bufferBlock.Complete(); await bufferBlock.Completion.ContinueWith(delegate { batchBlock.Complete(); }); Console.ReadKey(); }).GetAwaiter().GetResult(); }
static void Main(string[] args) { var batchBlock = new BatchBlock <int>(3); for (int i = 0; i < 10; i++) { batchBlock.Post(i); } batchBlock.Complete(); batchBlock.Post(99); // This will be ignored for (int i = 0; i < 5; i++) { if (batchBlock.TryReceive(out int[] result)) { Console.Write($"Received batch {i}: "); foreach (var item in result) { Console.Write($"{item} "); } Console.WriteLine(); } else { Console.WriteLine("The block completed."); break; } } Console.WriteLine("Finished!"); Console.ReadKey(); }
private static void TestBatchBlock() { //var batchBlock = new BatchBlock<int>(10); //for (int i = 0; i < 13; i++) //{ // batchBlock.Post(i); //} //batchBlock.Complete(); //Console.WriteLine("The sum of the elements in batch 1 is {0}.", batchBlock.Receive().Sum()); //Console.WriteLine("The sum of the elements in batch 2 is {0}.", batchBlock.Receive().Sum()); var bb = new BatchBlock <int>(3); var ab = new ActionBlock <int[]>(i => { string s = string.Empty; foreach (int m in i) { s += m + " "; } Console.WriteLine(s); }); bb.LinkTo(ab); for (int i = 0; i < 10; i++) { bb.Post(i); } bb.Complete(); }
public static async Task <List <string> > LoadFiles(ConcurrentDictionary <string, MemoryStream> streams) { List <string> _errors = new List <string>(); Queue <KeyValuePair <string, MemoryStream> > files = new Queue <KeyValuePair <string, MemoryStream> >(streams); var batchBlock = new BatchBlock <KeyValuePair <string, MemoryStream> >(75, new GroupingDataflowBlockOptions { BoundedCapacity = 100 }); var actionBlock = new ActionBlock <KeyValuePair <string, MemoryStream>[]>(t => { for (int i = 0; i < t.Length; i++) { var s = files.Dequeue(); try { DBReader reader = new DBReader(); DBEntry entry = reader.Read(s.Value, s.Key); if (entry != null) { var current = Entries.FirstOrDefault(x => x.FileName == entry.FileName && x.Build == entry.Build); if (current != null) { Entries.Remove(current); } Entries.Add(entry); if (!string.IsNullOrWhiteSpace(reader.ErrorMessage)) { _errors.Add(FormatError(s.Key, ErrorType.Warning, reader.ErrorMessage)); } } } catch (ConstraintException ex) { _errors.Add(FormatError(s.Key, ErrorType.Error, "Id column contains duplicates.")); } catch (Exception ex) { _errors.Add(FormatError(s.Key, ErrorType.Error, ex.Message)); } if (i % 100 == 0 && i > 0) { ForceGC(); } } ForceGC(); }); batchBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true }); foreach (KeyValuePair <string, MemoryStream> i in streams) { await batchBlock.SendAsync(i); // wait synchronously for the block to accept. } batchBlock.Complete(); await actionBlock.Completion; ForceGC(); return(_errors); }
// Adds random employee data to the database by using dataflow. // This method is similar to AddEmployees except that it uses batching // to add multiple employees to the database at a time. static void AddEmployeesBatched(string connectionString, int batchSize, int count) { // Create a BatchBlock<Employee> that holds several Employee objects and // then propagates them out as an array. BatchBlock <Employee> batchEmployees = new BatchBlock <Employee>(batchSize); // Create an ActionBlock<Employee[]> object that adds multiple // employee entries to the database. ActionBlock <Employee[]> insertEmployees = new ActionBlock <Employee[]>(a => { DatabaseUtilities.InsertEmployees(a, connectionString, "AddEmployeesBatched"); }); // Link the batch block to the action block. batchEmployees.LinkTo(insertEmployees); // When the batch block completes, set the action block also to complete. batchEmployees.Completion.ContinueWith(obj => { insertEmployees.Complete(); }); // Post several random Employee objects to the batch block. PostRandomEmployees(batchEmployees, count); // Set the batch block to the completed state and wait for // all insert operations to complete. batchEmployees.Complete(); insertEmployees.Completion.Wait(); }
private void BlockComplete() { _source.Complete(); _batchBlock.Complete(); _transformBlock.Complete(); _writer.Complete(); }
public SqlServerBackend(string connectionString, string collectorName, ISystemMetricsService systemMetrics, int retries = 3, int batchSize = 50) { _log = SuperCheapIOC.Resolve <ILog>(); _connectionString = connectionString; _collectorName = collectorName; _systemMetrics = systemMetrics; _retries = retries; InitialiseRetryHandling(); _batchBlock = new BatchBlock <GraphiteLine>(batchSize); _actionBlock = new ActionBlock <GraphiteLine[]>(p => SendToDB(p), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1 }); _batchBlock.LinkTo(_actionBlock); _batchBlock.Completion.ContinueWith(p => _actionBlock.Complete()); _actionBlock.Completion.ContinueWith(p => { _isActive = false; }); _completionTask = new Task(() => { _log.Info("SqlServerBackend - Completion has been signaled. Waiting for action block to complete."); _batchBlock.Complete(); _actionBlock.Completion.Wait(); }); }
public void Start() { const int numberChunks = 30; watch = new Stopwatch(); watch.Start(); for (int j = 1; j <= numberChunks; j++) { int collectionSize = 1000 * j; List <InputObject> collection = new List <InputObject>(collectionSize); for (int i = 0; i < collectionSize; i++) { collection.Add(new InputObject(i)); } tempBCB.Post(collection); } tempBCB.Complete(); Task.WhenAll(core1.transformBlock.Completion, core2.transformBlock.Completion).ContinueWith(_ => { batchBlock.Complete(); }); transformBlock.Completion.Wait(); Console.WriteLine("Completed"); Console.ReadLine(); }
public async Task BatchBlockWillCompleteTarget() { BatchBlock <int> bb = new BatchBlock <int>(batchSize: 2, dataflowBlockOptions: new GroupingDataflowBlockOptions() { BoundedCapacity = 3 }); TestTargetBlock <int[]> testTarget = new TestTargetBlock <int[]>(); testTarget.ConsumptionMode = DataflowMessageStatus.Accepted; bb.LinkTo(testTarget, PropagateCompletion); // Rapidly send 50 messages Task.WaitAll(Enumerable.Range(0, 50).Select((i) => bb.SendAsync(i)).ToArray(), BurstTimeout); bb.Complete(); // Completion should run to successful conclusion await Task.WhenAny(bb.Completion, Task.Delay(CompletionTimeout)); Assert.Equal(TaskStatus.RanToCompletion, bb.Completion.Status); // Assumption: BufferBlock should also have completed its target await Task.WhenAny(testTarget.Completion, Task.Delay(CompletionTimeout)); Assert.Equal(TaskStatus.RanToCompletion, testTarget.Completion.Status); // Assumption: we should have gotten 25 batches bool allMessagesReceived = await TaskUtils.PollWaitAsync(() => testTarget.MessagesConsumed.Count == 25, MessageArrivalTimeout); Assert.True(allMessagesReceived); }
/// <summary> /// GeneratePipeline_DataFlowSource_to_DataFlowDestination generates a TPL-DataFlowPipeline between two vertices of a graph. /// v_source.UserDefinedObjects[0] has to be Type of IDataFlowSource /// v_dest.UserDefinedObjects[0] has to be Type of IDataFlowDestination /// </summary> /// <param name="v_source"></param> /// <param name="v_dest"></param> /// <param name="ToCompleteCollection"></param> /// <param name="WatingForCompletitionCollection"></param> private void GeneratePipeline_DataFlowSource_to_DataFlowDestination(Vertex v_source, Vertex v_dest, ref List <object> ToCompleteCollection, ref List <object> WatingForCompletitionCollection, ref Dictionary <IDataFlowSource <DS>, object> DataFlowReaderCollection) { IDataFlowSource <DS> t_b_source = (IDataFlowSource <DS>)v_source.UserDefinedObjects[0]; IDataFlowDestination <DS> dest = (IDataFlowDestination <DS>)v_dest.UserDefinedObjects[0]; TransformBlock <DS, DS> t_b_dummy = new TransformBlock <DS, DS>(DS => { return(DS); } , new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = this.MaxDegreeOfParallelism }); ToCompleteCollection.Add(t_b_dummy); v_dest.UserDefinedObjects.Add(t_b_dummy); DataFlowReaderCollection.Add(t_b_source, t_b_dummy); var bacthBlock = new BatchBlock <DS>(BatchSize); var DataFlowDestinationBlock = new ActionBlock <DS[]>(outp => dest.WriteBatch(outp)); t_b_dummy.LinkTo(bacthBlock, linkOptions); bacthBlock.LinkTo(DataFlowDestinationBlock, linkOptions); t_b_dummy.Completion.ContinueWith(t => { bacthBlock.Complete(); }); bacthBlock.Completion.ContinueWith(t => { DataFlowDestinationBlock.Complete(); }); WatingForCompletitionCollection.Add(DataFlowDestinationBlock); }
static void ShowBatchBlock() { // <snippet7> // Create a BatchBlock<int> object that holds ten // elements per batch. var batchBlock = new BatchBlock <int>(10); // Post several values to the block. for (int i = 0; i < 13; i++) { batchBlock.Post(i); } // Set the block to the completed state. This causes // the block to propagate out any remaining // values as a final batch. batchBlock.Complete(); // Print the sum of both batches. Console.WriteLine("The sum of the elements in batch 1 is {0}.", batchBlock.Receive().Sum()); Console.WriteLine("The sum of the elements in batch 2 is {0}.", batchBlock.Receive().Sum()); /* Output: * The sum of the elements in batch 1 is 45. * The sum of the elements in batch 2 is 33. */ // </snippet7> }
private static async Task Synch(CancellationToken cancellationToken) { BatchBlock <int> syncBatch = new BatchBlock <int>(1); ActionBlock <int[]> syncAction = new ActionBlock <int[]>(async i => { Console.WriteLine("syncAction"); //exists pairs throw null; }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 }); _ = syncBatch.LinkTo(syncAction, new DataflowLinkOptions { PropagateCompletion = true }); await foreach (int i in AsyncEnumerable(cancellationToken)) { bool isMsgRecived = await syncBatch.SendAsync(i); if (isMsgRecived is false || syncAction.Completion.Status == TaskStatus.Faulted) { break; } } syncBatch.Complete(); await syncAction.Completion; }
public async Task TestPrecancellation() { var b = new BatchBlock <int>(42, new GroupingDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true), MaxNumberOfGroups = 1 }); Assert.Equal(expected: 42, actual: b.BatchSize); Assert.NotNull(b.LinkTo(DataflowBlock.NullTarget <int[]>())); Assert.False(b.Post(42)); Task <bool> t = b.SendAsync(42); Assert.True(t.IsCompleted); Assert.False(t.Result); int[] ignoredValue; IList <int[]> ignoredValues; Assert.False(b.TryReceive(out ignoredValue)); Assert.False(b.TryReceiveAll(out ignoredValues)); Assert.Equal(expected: 0, actual: b.OutputCount); Assert.NotNull(b.Completion); b.Complete(); // verify doesn't throw await Assert.ThrowsAnyAsync <OperationCanceledException>(() => b.Completion); }
public static async Task <List <string> > LoadFiles(IEnumerable <string> filenames) { ConcurrentBag <string> _errors = new ConcurrentBag <string>(); ConcurrentQueue <string> files = new ConcurrentQueue <string>(filenames.Distinct().OrderBy(x => x).ThenByDescending(x => Path.GetExtension(x))); string firstFile = files.First(); var batchBlock = new BatchBlock <string>(100, new GroupingDataflowBlockOptions { BoundedCapacity = 100 }); var actionBlock = new ActionBlock <string[]>(t => { for (int i = 0; i < t.Length; i++) { string file; files.TryDequeue(out file); try { DBReader reader = new DBReader(); DBEntry entry = reader.Read(file); if (entry != null) { var current = Entries.FirstOrDefault(x => x.FileName == entry.FileName && x.Build == entry.Build); if (current != null) { Entries.Remove(current); } Entries.Add(entry); if (file != firstFile) { entry.Detach(); } if (!string.IsNullOrWhiteSpace(reader.ErrorMessage)) { _errors.Add(FormatError(file, ErrorType.Warning, reader.ErrorMessage)); } } } catch (ConstraintException ex) { _errors.Add(FormatError(file, ErrorType.Error, "Id column contains duplicates.")); } catch (Exception ex) { _errors.Add(FormatError(file, ErrorType.Error, ex.Message)); } } ForceGC(); }); batchBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true }); foreach (string i in files) { await batchBlock.SendAsync(i); // wait synchronously for the block to accept. } batchBlock.Complete(); await actionBlock.Completion; files = null; return(_errors.ToList()); }
static void Main(string[] args) { var batchBlock = new BatchBlock <int>(3); for (int i = 0; i < 10; i++) { batchBlock.Post(i); } batchBlock.Complete(); batchBlock.Post(10); for (int i = 0; i < 5; i++) { if (batchBlock.TryReceive(out var result)) { Console.Write($"Received batch {i}: "); foreach (var r in result) { Console.Write(r + " "); } Console.WriteLine(); } else { Console.WriteLine("block finished"); break; } } Console.WriteLine("done"); Console.ReadKey(); }
public async Task TestNonGreedy() { var batch = new BatchBlock <int>(2, new GroupingDataflowBlockOptions { Greedy = false }); for (int trial = 0; trial < 2; trial++) { Task <bool> firstSend = batch.SendAsync(1 + trial); Assert.False(firstSend.IsCompleted); // should always pass, but due to race might not test what we really want it to Assert.Equal(expected: 0, actual: batch.OutputCount); // ditto Task <bool> secondSend = batch.SendAsync(3 + trial); Assert.Equal(expected: true, actual: await firstSend); Assert.Equal(expected: true, actual: await secondSend); Assert.Equal(expected: true, actual: await batch.OutputAvailableAsync()); Assert.Equal(expected: 1, actual: batch.OutputCount); int[] result = await batch.ReceiveAsync(); Assert.NotNull(result); Assert.Equal(expected: 2, actual: result.Length); Assert.Equal(expected: 1 + trial, actual: result[0]); Assert.Equal(expected: 3 + trial, actual: result[1]); } batch.Complete(); await batch.Completion; }
public void TestTriggerBatch_Nop() { const int Iters = 2; var b = new BatchBlock <int>(1); for (int i = 0; i < Iters; i++) { b.Post(i); int outputCount = b.OutputCount; Assert.Equal(expected: i + 1, actual: outputCount); b.TriggerBatch(); Assert.Equal(expected: outputCount, actual: b.OutputCount); } b = new BatchBlock <int>(1); Assert.Equal(expected: 0, actual: b.OutputCount); for (int i = 0; i < 2; i++) { b.TriggerBatch(); } for (int i = 0; i < 2; i++) { b.Complete(); b.TriggerBatch(); } Assert.Equal(expected: 0, actual: b.OutputCount); }
public Task DownloadThemAllAsync(IEnumerable <Uri> uris, ProcessResult processResult, byte maxThreads) { //Multiple Consumers var consumers = new ActionBlock <Uri[]>( //Handlers uriBatch => ConsumerAsync(uriBatch, processResult), //!SPOT: thread-safety should be guaranteed by us new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxThreads }); var sharedUris = new BatchBlock <Uri>(2 * maxThreads); //Command //This is is the link between the pipeline's command and handlers //!SPOT: Link together producer to consumers sharedUris.LinkTo(consumers, new DataflowLinkOptions { PropagateCompletion = true }); //!SPOT: without this option the consumers will never finish //Single Producer Task.Run(async() => { await ProducerAsync(sharedUris, uris); //Signaling producing is over sharedUris.Complete(); }); //Waiting for all consumers to finish return(consumers.Completion); }
public async Task TestLinkToOptions() { const int Messages = 2; foreach (bool append in DataflowTestHelpers.BooleanValues) { var bb = new BatchBlock <int>(1); var values = new int[Messages][]; var targets = new ActionBlock <int[]> [Messages]; for (int i = 0; i < Messages; i++) { int slot = i; targets[i] = new ActionBlock <int[]>(item => values[slot] = item); bb.LinkTo(targets[i], new DataflowLinkOptions { MaxMessages = 1, Append = append }); } bb.PostRange(0, Messages); bb.Complete(); await bb.Completion; for (int i = 0; i < Messages; i++) { targets[i].Complete(); await targets[i].Completion; Assert.Equal( expected: append ? i : Messages - i - 1, actual: values[i][0]); } } }
public async Task BufferBlocksToBatchNonGreedyToAction() { var inputs = Enumerable.Range(0, 1).Select(_ => new BufferBlock <int>()).ToList(); var b = new BatchBlock <int>(inputs.Count); int completedCount = 0; var c = new ActionBlock <int[]>(i => completedCount++); foreach (var input in inputs) { input.LinkTo(b); } var ignored = Task.WhenAll(inputs.Select(s => s.Completion)).ContinueWith( _ => b.Complete(), CancellationToken.None, TaskContinuationOptions.None, TaskScheduler.Default); b.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true }); for (int i = 0; i < Iterations; i++) { inputs[i % inputs.Count].Post(i); } foreach (var input in inputs) { input.Complete(); } await c.Completion; Assert.Equal(expected: Iterations / b.BatchSize, actual: completedCount); }
private void ConfigureEntity <T>(Func <Message, T> action, Action <T[]> execution, int batchSize, DataflowLinkOptions linkOptions) where T : class { var transformBlock = new TransformBlock <Message, T>(action); var batchBlock = new BatchBlock <T>(batchSize); var actionBlock = new ActionBlock <T[]>(m => { var temp = m.Where(x => x != null).ToArray(); var entity = temp.GetType().Name; _logger.LogDebug($"Bulk insert {entity} - {temp.Length}"); execution.Invoke(temp); _logger.LogInformation($"Process finished for {entity}"); }); batchBlock.LinkTo(actionBlock, linkOptions); batchBlock.Completion.ContinueWith(delegate { actionBlock.Complete(); }); transformBlock.LinkTo(batchBlock, linkOptions); transformBlock.Completion.ContinueWith(delegate { batchBlock.Complete(); }); var bufferBlock = new BufferBlock <Message>(); bufferBlock.LinkTo(transformBlock, linkOptions, m => m != null && m.Type == typeof(T).Name); _bufferBlocks.Add(typeof(T).Name, bufferBlock); }
/// <summary> /// The following basic example posts several Int32 values to a BatchBlock<T> object /// that holds ten elements in a batch. To guarantee that all values propagate out of the /// BatchBlock<T>, this example calls the Complete method. The Complete method sets the /// BatchBlock<T> object to the completed state, and therefore, the BatchBlock<T> object /// propagates out any remaining elements as a final batch. /// </summary> public static void Run() { // Create a BatchBlock<int> object that holds ten // elements per batch. // Provides a dataflow block that batches inputs into arrays // Below every batch will have 10 items. BatchBlock <int> batchBlock = new BatchBlock <int>(10); Console.WriteLine("BatchBlock<int> batchBlock = new BatchBlock<int>(10);"); // Post several values to the block. for (int i = 0; i < 13; i++) { batchBlock.Post(i); Console.WriteLine("Posted to batchBlock {0}.", i); } // Set the block to the completed state. This causes // the block to propagate out any any remaining // values as a final batch. batchBlock.Complete(); Console.WriteLine("batchBlock.Complete();"); // Print the sum of both batches. int length1 = batchBlock.Receive().Length; int length2 = batchBlock.Receive().Length; Console.WriteLine("The total element count in batch-1 is {0}.", length1); Console.WriteLine("The total element count in batch-2 is {0}.", length2); }
public static void Main(string[] args) { var participants = Participant.Generate(100000, 100000, CompanyId).ToArray(); var actionBatchBlock = new BatchBlock <ActionParticipant>(7500); var insertActionParticipants = new ActionBlock <ActionParticipant[]>(items => { var bulkInsertAsync = DataImporter.BulkInsertAsync(items); Inserted += items.Length; Console.WriteLine($"Inserted {Inserted} out of { ActionCount}"); return(bulkInsertAsync); }); actionBatchBlock.LinkTo(insertActionParticipants); actionBatchBlock.Completion.ContinueWith(delegate { insertActionParticipants.Complete(); }); foreach (var actionParticipant in ActionParticipant.Generate(participants, ActionCount, 30000, CompanyId)) { actionBatchBlock.Post(actionParticipant); } actionBatchBlock.Complete(); insertActionParticipants.Completion.Wait(); }
public void Run() { var batchBlock = new BatchBlock <int>(10); for (int i = 0; i < 20; i++) { batchBlock.Post(i); } batchBlock.Complete(); batchBlock.Post(10); for (int i = 0; i < 5; i++) { int[] result; if (batchBlock.TryReceive(out result)) { Console.Write($"Received batch {i}:"); foreach (var r in result) { Console.Write(r + " "); } Console.Write("\n"); } else { Console.WriteLine("The block finished"); break; } } }
public void Configure(string collectorName, XElement configElement, ISystemMetricsService systemMetrics) { _systemMetrics = systemMetrics; var config = new SqlServerConfiguration(configElement.Attribute("connectionString").Value, configElement.ToInt("writeBatchSize")); _connectionString = config.ConnectionString; _collectorName = collectorName; _retries = config.Retries; InitialiseRetryHandling(); _batchBlock = new BatchBlock <GraphiteLine>(config.WriteBatchSize); _actionBlock = new ActionBlock <GraphiteLine[]>(p => SendToDB(p), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 }); _batchBlock.LinkTo(_actionBlock); _batchBlock.Completion.ContinueWith(p => _actionBlock.Complete()); _actionBlock.Completion.ContinueWith(p => { IsActive = false; }); Completion = new Task(() => { _log.Info("SqlServerBackend - Completion has been signaled. Waiting for action block to complete."); _batchBlock.Complete(); _actionBlock.Completion.Wait(); }); }
// A BatchBlock takes in multiple types of values and outputs those values joined together (amount specified in the constructor) private static async Task SimpleDemoAsync() { Console.WriteLine("BatchBlockDemo has started!"); var block = new BatchBlock <int>(3); // batch messages of type int, into 3 for (int i = 0; i < 10; i++) { block.Post(i); } block.Complete(); // No mo data. while (await block.OutputAvailableAsync().ConfigureAwait(false)) { var output = await block.ReceiveAsync().ConfigureAwait(false); Console.WriteLine($"BatchBlock BatchOutput: {string.Join(",", output)}"); Console.WriteLine($"BatchBlock OutputCount: {block.OutputCount}"); } // wait for completion. await block.Completion.ConfigureAwait(false); Console.WriteLine("Finished!"); Console.ReadKey(); }
static async Task<long> skynetTpl( long num, long size, long div ) { BatchBlock<long> source = new BatchBlock<long>( 1024 ); long sum = 0; ActionBlock<long[]> actAggregate = new ActionBlock<long[]>( vals => sum += vals.Sum(), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1, SingleProducerConstrained = true } ); source.LinkTo( actAggregate, new DataflowLinkOptions() { PropagateCompletion = true } ); skynetTplRecursion( source, num, size, div ); source.Complete(); await actAggregate.Completion; return sum; }
public async Task TestNonGreedyLostMessages() { foreach (int batchSize in new[] { 2, 5 }) { var batch = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions { Greedy = false }); var buffers = Enumerable.Range(0, batchSize - 1).Select(_ => new BufferBlock<int>()).ToList(); var tcs = new TaskCompletionSource<bool>(); int remaining = buffers.Count; // Offer the batch almost all the messages it needs, but have them consumed by someone else foreach (var buffer in buffers) { buffer.LinkTo(batch); buffer.LinkTo(new ActionBlock<int>(i => { if (Interlocked.Decrement(ref remaining) == 0) { tcs.SetResult(true); } })); buffer.Post(42); } await tcs.Task; // Now offer from another set of sources that won't lose them buffers = Enumerable.Range(0, batchSize).Select(_ => new BufferBlock<int>()).ToList(); foreach (var buffer in buffers) { buffer.LinkTo(batch); buffer.Post(43); buffer.Complete(); } // Wait until all the messages are consumed await Task.WhenAll(from buffer in buffers select buffer.Completion); int[] results = await batch.ReceiveAsync(); Assert.Equal(expected: 0, actual: batch.OutputCount); batch.Complete(); await batch.Completion; } }
public async Task BatchGreedyToAction() { var b = new BatchBlock<int>(1); int completedCount = 0; var c = new ActionBlock<int[]>(i => completedCount++); b.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true }); b.PostRange(0, Iterations); b.Complete(); await c.Completion; Assert.Equal(expected: Iterations / b.BatchSize, actual: completedCount); }
public async Task TestProducerConsumer() { foreach (TaskScheduler scheduler in new[] { TaskScheduler.Default, new ConcurrentExclusiveSchedulerPair().ExclusiveScheduler }) foreach (int maxMessagesPerTask in new[] { DataflowBlockOptions.Unbounded, 1, 2 }) foreach (int boundedCapacity in new[] { DataflowBlockOptions.Unbounded, 2, 3 }) foreach (int batchSize in new[] { 1, 2 }) { const int Messages = 50; var bb = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions { BoundedCapacity = boundedCapacity, MaxMessagesPerTask = maxMessagesPerTask, TaskScheduler = scheduler }); await Task.WhenAll( Task.Run(async delegate { // consumer int i = 0; while (await bb.OutputAvailableAsync()) { int[] items = await bb.ReceiveAsync(); Assert.Equal(expected: batchSize, actual: items.Length); for (int j = 0; j < items.Length; j++) { Assert.Equal(expected: i + j, actual: items[j]); } i += batchSize; } }), Task.Run(async delegate { // producer for (int i = 0; i < Messages; i++) { await bb.SendAsync(i); } bb.Complete(); })); } }
public async Task TestLinkToOptions() { const int Messages = 2; foreach (bool append in DataflowTestHelpers.BooleanValues) { var bb = new BatchBlock<int>(1); var values = new int[Messages][]; var targets = new ActionBlock<int[]>[Messages]; for (int i = 0; i < Messages; i++) { int slot = i; targets[i] = new ActionBlock<int[]>(item => values[slot] = item); bb.LinkTo(targets[i], new DataflowLinkOptions { MaxMessages = 1, Append = append }); } bb.PostRange(0, Messages); bb.Complete(); await bb.Completion; for (int i = 0; i < Messages; i++) { targets[i].Complete(); await targets[i].Completion; Assert.Equal( expected: append ? i : Messages - i - 1, actual: values[i][0]); } } }
public async Task BufferBlocksToBatchNonGreedyToAction() { var inputs = Enumerable.Range(0, 1).Select(_ => new BufferBlock<int>()).ToList(); var b = new BatchBlock<int>(inputs.Count); int completedCount = 0; var c = new ActionBlock<int[]>(i => completedCount++); foreach (var input in inputs) input.LinkTo(b); var ignored = Task.WhenAll(inputs.Select(s => s.Completion)).ContinueWith( _ => b.Complete(), CancellationToken.None, TaskContinuationOptions.None, TaskScheduler.Default); b.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true }); for (int i = 0; i < Iterations; i++) { inputs[i % inputs.Count].Post(i); } foreach (var input in inputs) input.Complete(); await c.Completion; Assert.Equal(expected: Iterations / b.BatchSize, actual: completedCount); }
public async Task TestBoundedReceives() { for (int test = 0; test < 4; test++) { var bb = new BatchBlock<int>(1, new GroupingDataflowBlockOptions { BoundedCapacity = 1 }); Assert.True(bb.Post(0)); int[] item; const int sends = 5; for (int i = 1; i <= sends; i++) { Task<bool> send = bb.SendAsync(i); Assert.True(await bb.OutputAvailableAsync()); // wait for previously posted/sent item switch (test) { case 0: IList<int[]> items; Assert.True(bb.TryReceiveAll(out items)); Assert.Equal(expected: 1, actual: items.Count); Assert.Equal(expected: 1, actual: items[0].Length); Assert.Equal(expected: i - 1, actual: items[0][0]); break; case 1: Assert.True(bb.TryReceive(f => true, out item)); Assert.Equal(expected: 1, actual: item.Length); Assert.Equal(expected: i - 1, actual: item[0]); break; case 2: Assert.False(bb.TryReceive(f => f.Length == 1 && f[0] == i, out item)); Assert.True(bb.TryReceive(f => f.Length == 1 && f[0] == i - 1, out item)); Assert.Equal(expected: 1, actual: item.Length); Assert.Equal(expected: i - 1, actual: item[0]); break; case 3: item = await bb.ReceiveAsync(); Assert.Equal(expected: 1, actual: item.Length); Assert.Equal(expected: i - 1, actual: item[0]); break; } } // Receive remaining item item = await bb.ReceiveAsync(); Assert.Equal(expected: 1, actual: item.Length); Assert.Equal(expected: sends, actual: item[0]); bb.Complete(); await bb.Completion; } }
public async Task TestCompletionWithBufferedItems() { var b = new BatchBlock<int>(5); b.PostRange(0, 3); b.Complete(); await b.OutputAvailableAsync(); Assert.Equal(expected: 1, actual: b.OutputCount); int[] items = await b.ReceiveAsync(); Assert.Equal(expected: 3, actual: items.Length); await b.Completion; }
public async Task TestNonGreedy() { var batch = new BatchBlock<int>(2, new GroupingDataflowBlockOptions { Greedy = false }); for (int trial = 0; trial < 2; trial++) { Task<bool> firstSend = batch.SendAsync(1 + trial); Assert.False(firstSend.IsCompleted); // should always pass, but due to race might not test what we really want it to Assert.Equal(expected: 0, actual: batch.OutputCount); // ditto Task<bool> secondSend = batch.SendAsync(3 + trial); Assert.Equal(expected: true, actual: await firstSend); Assert.Equal(expected: true, actual: await secondSend); Assert.Equal(expected: true, actual: await batch.OutputAvailableAsync()); Assert.Equal(expected: 1, actual: batch.OutputCount); int[] result = await batch.ReceiveAsync(); Assert.NotNull(result); Assert.Equal(expected: 2, actual: result.Length); Assert.Equal(expected: 1 + trial, actual: result[0]); Assert.Equal(expected: 3 + trial, actual: result[1]); } batch.Complete(); await batch.Completion; }
internal static bool BatchGreedyToAction() { const int ITERS = 2; var b = new BatchBlock<int>(1); int completedCount = 0; var c = new ActionBlock<int[]>(i => completedCount++); b.LinkWithCompletion(c); for (int i = 0; i < ITERS; i++) b.Post(i); b.Complete(); c.Completion.Wait(); return completedCount == ITERS / b.BatchSize; }
public async Task TestPrecancellation() { var b = new BatchBlock<int>(42, new GroupingDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true), MaxNumberOfGroups = 1 }); Assert.Equal(expected: 42, actual: b.BatchSize); Assert.NotNull(b.LinkTo(DataflowBlock.NullTarget<int[]>())); Assert.False(b.Post(42)); Task<bool> t = b.SendAsync(42); Assert.True(t.IsCompleted); Assert.False(t.Result); int[] ignoredValue; IList<int[]> ignoredValues; Assert.False(b.TryReceive(out ignoredValue)); Assert.False(b.TryReceiveAll(out ignoredValues)); Assert.Equal(expected: 0, actual: b.OutputCount); Assert.NotNull(b.Completion); b.Complete(); // verify doesn't throw await Assert.ThrowsAnyAsync<OperationCanceledException>(() => b.Completion); }
internal static bool BufferBlocksToBatchNonGreedyToAction() { const int ITERS = 2; var inputs = Enumerable.Range(0, 1).Select(_ => new BufferBlock<int>()).ToList(); var b = new BatchBlock<int>(inputs.Count); int completedCount = 0; var c = new ActionBlock<int[]>(i => completedCount++); foreach (var input in inputs) input.LinkTo(b); Task.Factory.ContinueWhenAll(inputs.Select(i => i.Completion).ToArray(), _ => b.Complete()); b.LinkWithCompletion(c); for (int i = 0; i < ITERS; i++) { inputs[i % inputs.Count].Post(i); } foreach (var input in inputs) input.Complete(); c.Completion.Wait(); return (completedCount == ITERS / b.BatchSize); }
public void TestTriggerBatch_Nop() { const int Iters = 2; var b = new BatchBlock<int>(1); for (int i = 0; i < Iters; i++) { b.Post(i); int outputCount = b.OutputCount; Assert.Equal(expected: i + 1, actual: outputCount); b.TriggerBatch(); Assert.Equal(expected: outputCount, actual: b.OutputCount); } b = new BatchBlock<int>(1); Assert.Equal(expected: 0, actual: b.OutputCount); for (int i = 0; i < 2; i++) { b.TriggerBatch(); } for (int i = 0; i < 2; i++) { b.Complete(); b.TriggerBatch(); } Assert.Equal(expected: 0, actual: b.OutputCount); }
public void Init() { _massMail = new MassMail(Config.BlockSize, Config.UserAgent, Config.ConnectionString, Config.Mode); _templateCache = new ConcurrentDictionary<long, Lazy<Template>>(); _attachmentCache = new ConcurrentDictionary<long, Lazy<Attach>>(); _dkimSignerCache = new ConcurrentDictionary<string, DkimSigner>(); _domailKeySignerCache = new ConcurrentDictionary<string, DomainKeySigner>(); //Get all private keys GetDkimSigners(); //*** Create pipeline *** //Create TransformBlock that gets table of client data and make a list of objects from them. _parseXmlDataBlock = new TransformBlock<DataTable, List<Mail>>(sendData => ParseXmlData(sendData), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Config.ParseXmlMaxdop, BoundedCapacity = Config.ParseXmlBufferSize }); //Create TransformBlock that gets a list of client objects, send them email, and stores result in DataTable. _sendEmailsBlock = new TransformBlock<List<Mail>, DataTable>(mails => SendEmails(_massMail, mails), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Config.SendEmailsMaxdop, BoundedCapacity = Config.SendEmailsMaxdop }); //Create BatchBlock that holds several DataTable and then propagates them out as an array. _batchResultBlock = new BatchBlock<DataTable>(Config.BatchSize, new GroupingDataflowBlockOptions { BoundedCapacity = Config.BatchSize }); //Create ActionBlock that writes result into DB _writeResultsBlock = new ActionBlock<DataTable[]>(results => WriteResults(_massMail, results), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 }); //*** Build pipeline *** // POST --> _parseXmlDataBlock --> _sendEmailsBlock --> _batchResultBlock --> _writeResultsBlock _parseXmlDataBlock.LinkTo(_sendEmailsBlock); _sendEmailsBlock.LinkTo(_batchResultBlock); _batchResultBlock.LinkTo(_writeResultsBlock); _parseXmlDataBlock.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)_sendEmailsBlock).Fault(t.Exception); else _sendEmailsBlock.Complete(); }); _sendEmailsBlock.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)_batchResultBlock).Fault(t.Exception); else _batchResultBlock.Complete(); }); _batchResultBlock.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)_writeResultsBlock).Fault(t.Exception); else _writeResultsBlock.Complete(); }); }
private static bool TestTriggerBatch(int boundedCapacity) { bool passed = true; // Test greedy with batch size of 1 (force should always be a nop) { bool localPassed = true; const int ITERS = 2; var b = new BatchBlock<int>(1, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity }); for (int i = 0; i < ITERS; i++) { b.Post(i); int outputCount = b.OutputCount; b.TriggerBatch(); localPassed &= outputCount == b.OutputCount; } localPassed &= b.OutputCount == ITERS; for (int i = 0; i < ITERS; i++) { var arr = b.Receive(); localPassed &= arr.Length == 1 && arr[0] == i; } Assert.True(localPassed, string.Format("greedy with batch size of 1 - {0}", localPassed ? "Passed" : "FAILED")); } // Test greedy with varying batch sizes and smaller queued numbers { bool localPassed = true; foreach (var batchSize in new[] { 3 }) { foreach (var queuedBeforeTrigger in new[] { 1, batchSize - 1 }) { var b = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity }); for (int p = 1; p <= queuedBeforeTrigger; p++) b.Post(p); localPassed &= b.OutputCount == 0; b.TriggerBatch(); b.OutputAvailableAsync().Wait(); // The previous batch is triggered asynchronously when non-Unbounded BoundedCapacity is provided localPassed &= b.OutputCount == 1 && b.Receive().Length == queuedBeforeTrigger; for (int j = 0; j < batchSize; j++) { localPassed &= b.OutputCount == 0; b.Post(j); } localPassed &= b.OutputCount == 1; } } Assert.True(localPassed, string.Format("greedy with varying batch sizes - {0}", localPassed ? "Passed" : "FAILED")); passed &= localPassed; } // Test greedy with empty queue { bool localPassed = true; foreach (var batchSize in new[] { 1 }) { var b = new BatchBlock<int>(batchSize, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity }); for (int i = 0; i < 2; i++) b.TriggerBatch(); localPassed &= b.OutputCount == 0; } Assert.True(localPassed, string.Format("greedy with empty queue - {0}", localPassed ? "Passed" : "FAILED")); passed &= localPassed; } // Test greedy after decline { bool localPassed = true; { var b = new BatchBlock<int>(2, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity }); localPassed &= b.OutputCount == 0; b.Complete(); localPassed &= b.OutputCount == 0; b.TriggerBatch(); localPassed &= b.OutputCount == 0; } { var b = new BatchBlock<int>(2, new GroupingDataflowBlockOptions() { BoundedCapacity = boundedCapacity }); localPassed &= b.OutputCount == 0; b.Post(1); localPassed &= b.OutputCount == 0; b.Complete(); localPassed &= b.OutputCount == 1; b.TriggerBatch(); localPassed &= b.OutputCount == 1; } Assert.True(localPassed, string.Format("greedy after decline - {0}", localPassed ? "Passed" : "FAILED")); passed &= localPassed; } // Test greedy after canceled { bool localPassed = true; { var cts = new CancellationTokenSource(); var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, BoundedCapacity = boundedCapacity }; var b = new BatchBlock<int>(2, dbo); localPassed &= b.OutputCount == 0; cts.Cancel(); localPassed &= b.OutputCount == 0; b.TriggerBatch(); localPassed &= b.OutputCount == 0; } { var cts = new CancellationTokenSource(); var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, BoundedCapacity = boundedCapacity }; var b = new BatchBlock<int>(2, dbo); localPassed &= b.OutputCount == 0; b.Post(1); localPassed &= b.OutputCount == 0; cts.Cancel(); localPassed &= b.OutputCount == 0; b.TriggerBatch(); localPassed &= b.OutputCount == 0; } Assert.True(localPassed, string.Format("greedy after canceled - {0}", localPassed ? "Passed" : "FAILED")); passed &= localPassed; } // Test greedy with MaxNumberOfGroups == 1 { bool localPassed = true; var b = new BatchBlock<int>(2, new GroupingDataflowBlockOptions { MaxNumberOfGroups = 1, BoundedCapacity = boundedCapacity }); b.Post(1); localPassed &= b.OutputCount == 0; b.TriggerBatch(); b.OutputAvailableAsync().Wait(); // The previous batch is triggered asynchronously when non-Unbounded BoundedCapacity is provided localPassed &= b.OutputCount == 1; localPassed &= b.Post(2) == false; b.TriggerBatch(); localPassed &= b.OutputCount == 1; Assert.True(localPassed, string.Format("greedy with MaxNumberOfGroups == 1 - {0}", localPassed ? "Passed" : "FAILED")); passed &= localPassed; } // Test non-greedy with no queued or postponed messages { bool localPassed = true; var dbo = new GroupingDataflowBlockOptions { Greedy = false, BoundedCapacity = boundedCapacity }; var b = new BatchBlock<int>(3, dbo); localPassed &= b.OutputCount == 0; b.TriggerBatch(); localPassed &= b.OutputCount == 0; Assert.True(localPassed, string.Format("non-greedy with no mesages - {0}", localPassed ? "Passed" : "FAILED")); passed &= localPassed; } // Test non-greedy with no queued but postponed messages { bool localPassed = true; var dbo = new GroupingDataflowBlockOptions { Greedy = false, BoundedCapacity = boundedCapacity }; const int BATCH_SIZE = 10; for (int numPostponedMessages = 1; numPostponedMessages < BATCH_SIZE; numPostponedMessages++) { var b = new BatchBlock<int>(BATCH_SIZE, dbo); localPassed &= b.OutputCount == 0; for (int i = 0; i < numPostponedMessages; i++) localPassed &= !b.SendAsync(i).IsCompleted; b.TriggerBatch(); var output = b.Receive(); localPassed &= output.Length == numPostponedMessages; for (int i = 0; i < output.Length; i++) localPassed &= output[i] == i; localPassed &= b.OutputCount == 0; b.TriggerBatch(); localPassed &= b.OutputCount == 0; } Assert.True(localPassed, string.Format("non-greedy with postponed, no queued - {0}", localPassed ? "Passed" : "FAILED")); passed &= localPassed; } return passed; }
public void NonGreedyCompleteDoesnTriggerBatchTest () { var scheduler = new TestScheduler (); var block = new BatchBlock<int> (2, new GroupingDataflowBlockOptions { Greedy = false, TaskScheduler = scheduler }); var source = new BufferBlock<int> (new DataflowBlockOptions { TaskScheduler = scheduler }); Assert.IsNotNull (source.LinkTo (block)); Assert.IsTrue (source.Post (1)); block.Complete (); int[] batch; Assert.IsFalse (block.TryReceive (out batch)); Assert.IsTrue (block.Completion.Wait (1000)); }
public void RunBatchBlockConformanceTests() { bool localPassed; // Greedy batching { localPassed = true; const int NUM_MESSAGES = 1; const int BATCH_SIZE = 1; var batch = new BatchBlock<int>(BATCH_SIZE); for (int i = 0; i < NUM_MESSAGES * BATCH_SIZE; i++) batch.Post(i); for (int i = 0; i < NUM_MESSAGES; i++) { int[] result = batch.Receive(); localPassed &= result.Length == BATCH_SIZE; for (int j = 0; j < result.Length - 1; j++) { localPassed &= (result[j] + 1 == result[j + 1]); } } Assert.True(localPassed, string.Format("{0}: Greedy batching", localPassed ? "Success" : "Failure")); } // Non-greedy batching with BATCH_SIZE sources used repeatedly { localPassed = true; const int NUM_MESSAGES = 1; const int BATCH_SIZE = 1; var batch = new BatchBlock<int>(BATCH_SIZE, new GroupingDataflowBlockOptions { Greedy = false }); var buffers = Enumerable.Range(0, BATCH_SIZE).Select(_ => new BufferBlock<int>()).ToList(); foreach (var buffer in buffers) buffer.LinkTo(batch); int prevSum = -1; for (int i = 0; i < NUM_MESSAGES; i++) { for (int j = 0; j < BATCH_SIZE; j++) buffers[j].Post(i); int sum = batch.Receive().Sum(); localPassed &= (sum > prevSum); prevSum = sum; } Assert.True(localPassed, string.Format("{0}: Non-greedy batching with BATCH_SIZE sources used repeatedly", localPassed ? "Success" : "Failure")); } // Non-greedy batching with BATCH_SIZE * NUM_MESSAGES sources { localPassed = true; const int NUM_MESSAGES = 1; const int BATCH_SIZE = 2; var batch = new BatchBlock<int>(BATCH_SIZE, new GroupingDataflowBlockOptions { Greedy = false }); var buffers = Enumerable.Range(0, BATCH_SIZE * NUM_MESSAGES).Select(_ => new BufferBlock<int>()).ToList(); foreach (var buffer in buffers) { buffer.LinkTo(batch); buffer.Post(1); } for (int i = 0; i < NUM_MESSAGES; i++) { localPassed &= batch.Receive().Sum() == BATCH_SIZE; } Assert.True(localPassed, string.Format("{0}: Non-greedy batching with N*M sources", localPassed ? "Success" : "Failure")); } // Non-greedy batching with missed messages { localPassed = true; const int BATCH_SIZE = 2; var batch = new BatchBlock<int>(BATCH_SIZE, new GroupingDataflowBlockOptions { Greedy = false }); var buffers = Enumerable.Range(0, BATCH_SIZE - 1).Select(_ => new BufferBlock<int>()).ToList(); using (var ce = new CountdownEvent(BATCH_SIZE - 1)) { foreach (var buffer in buffers) { buffer.LinkTo(batch); buffer.LinkTo(new ActionBlock<int>(i => ce.Signal())); buffer.Post(42); } ce.Wait(); } buffers = Enumerable.Range(0, BATCH_SIZE).Select(_ => new BufferBlock<int>()).ToList(); foreach (var buffer in buffers) { buffer.LinkTo(batch); buffer.Post(42); buffer.Complete(); } localPassed &= Task.WaitAll(buffers.Select(b => b.Completion).ToArray(), 2000); Assert.True(localPassed, string.Format("{0}: Non-greedy batching with missed messages", localPassed ? "Success" : "Failure")); } // Test using a precanceled token { localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new GroupingDataflowBlockOptions { CancellationToken = cts.Token, MaxNumberOfGroups = 1 }; var b = new BatchBlock<int>(42, dbo); int[] ignoredValue; IList<int[]> ignoredValues; localPassed &= b.BatchSize == 42; localPassed &= b.LinkTo(new ActionBlock<int[]>(delegate { })) != null; localPassed &= b.SendAsync(42).Result == false; localPassed &= b.TryReceiveAll(out ignoredValues) == false; localPassed &= b.Post(42) == false; localPassed &= b.OutputCount == 0; localPassed &= b.TryReceive(out ignoredValue) == false; localPassed &= b.Completion != null; b.Complete(); } catch (Exception) { localPassed = false; } Assert.True(localPassed, string.Format("{0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure")); } // Test completing block while still items buffered { localPassed = true; var b = new BatchBlock<int>(5); b.Post(1); b.Post(2); b.Post(3); b.Complete(); localPassed &= b.Receive().Length == 3; Assert.True(localPassed, string.Format("{0}: Makes batches of remaining items", localPassed ? "Success" : "Failure")); } }
private static bool TestTriggerBatchRacingWithComplete(bool greedy) { bool passed = true; const int batchSize = 2; const int iterations = 1; const int waitTimeout = 100; var dbo = new GroupingDataflowBlockOptions { Greedy = greedy }; for (int iter = 0; iter < iterations; iter++) { bool localPassed = true; var sendAsyncTasks = new Task<bool>[batchSize - 1]; var racerReady = new ManualResetEventSlim(); int[] output1 = null; int[] output2 = null; // Blocks var batch = new BatchBlock<int>(batchSize, dbo); var terminator = new ActionBlock<int[]>(x => { if (output1 == null) output1 = x; else output2 = x; }); batch.LinkTo(terminator); // Queue up batchSize-1 input items for (int i = 0; i < batchSize - 1; i++) sendAsyncTasks[i] = batch.SendAsync(i); var racer = Task.Factory.StartNew(() => { racerReady.Set(); batch.Complete(); }); // Wait for the racer to get ready and trigger localPassed &= racerReady.Wait(waitTimeout); batch.TriggerBatch(); Assert.True(localPassed, "The racer task FAILED to start."); if (localPassed) { // Wait for the SendAsync tasks to complete localPassed &= Task.WaitAll(sendAsyncTasks, waitTimeout); Assert.True(localPassed, "SendAsync tasks FAILED to complete"); } // Do this verification only in greedy mode, because non-greedy is non-deterministic if (greedy) { // Wait for a batch to be produced if (localPassed) { localPassed &= SpinWait.SpinUntil(() => output1 != null, waitTimeout); Assert.True(localPassed, "FAILED to produce a batch"); } if (localPassed) { //Verify the number of input items propagated localPassed &= output1.Length == batchSize - 1; Assert.True(localPassed, string.Format("FAILED to propagate {0} input items. count1={1}", batchSize, output1.Length)); } } // Wait for the block to complete if (localPassed) { localPassed &= batch.Completion.Wait(waitTimeout); Assert.True(localPassed, "The block FAILED to complete"); } // There should never be a second batch produced if (localPassed) { localPassed &= output2 == null; Assert.True(localPassed, "FAILED not to produce a second batch"); } passed &= localPassed; if (!localPassed) { Assert.True(localPassed, string.Format("Iteration={0}", iter)); Assert.True(localPassed, string.Format("Count1={0}", output1 == null ? "null" : output1.Length.ToString())); break; } } return passed; }
public void CompleteTriggersBatchTest () { var block = new BatchBlock<int> (2); Assert.IsTrue (block.Post (1)); block.Complete (); CollectionAssert.AreEqual (new[] { 1 }, block.Receive (TimeSpan.FromMilliseconds (2000))); Assert.IsTrue (block.Completion.Wait (1000)); }
private async void StartRefreshStationsAsync() { IsStationWorkerRunning = true; while (refreshingPool.Count > 0) { await Task.Delay(15000).ConfigureAwait(false); if (CrossConnectivity.Current.IsConnected) { BatchBlock<Station> batchBlock = new BatchBlock<Station>(5); var actionBlock = new ActionBlock<Station[]>( async stations => { foreach (var station in stations) { if (await station.Contract.RefreshAsync(station)) { if (station.IsUiRefreshNeeded) { StationRefreshed?.Invoke(station, EventArgs.Empty); } } } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5 }); batchBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true }); foreach (var station in refreshingPool) { await batchBlock.SendAsync(station); // wait synchronously for the block to accept. } batchBlock.Complete(); actionBlock.Completion.Wait(15000); } } IsStationWorkerRunning = false; }