public async Task TimedBatchBlock_Example()
{
// setup timed batch block
var timedBatchBlock = new TimedBatchBlock <Customer>(new TimedBackBlockConfig());
timedBatchBlock.SetHandler(async customers => await PerformAction(customers, 1, 1));
List <Task> tasks = new List <Task>();
var bulkService = new BulkCustomerDataService(95);
await foreach (var importCustomer in bulkService.GetCustomersFromImportAsync())
{
// this is designed to be executed in many separate threads (using rebus typically)
// if you await here it will wait the full 10 seconds for each message
tasks.Add(timedBatchBlock.Queue(importCustomer.ToCustomer()));
}
// timed block will still have the last 5 records queued up waiting for more
await Task.Delay(1000);
Assert.AreEqual(9, _performActionCount, "Batch block should not be finished yet.");
await Task.Delay(2000);
Assert.AreEqual(10, _performActionCount, "Batch block should be done.");
Task.WaitAll(tasks.ToArray());
}
public async Task Save_OneAtATime_Parallel()
{
_currentTestCount = _customerCount / 100; // too slow to do full count
var bulkDataService = new BulkCustomerDataService(_currentTestCount);
var saveCount = 0;
// this will load the entire list into memory
var importCustomers = await bulkDataService.GetCustomersFromImportAsync().ToListAsync();
Parallel.ForEach(importCustomers,
new ParallelOptions()
{
MaxDegreeOfParallelism = 10
}, importCustomer =>
{
var customer = importCustomer.ToCustomer();
// parallel.foreach does not support async
_customerService.SaveCustomer(customer).Wait();
Interlocked.Increment(ref saveCount);
});
Assert.AreEqual(_currentTestCount, saveCount);
}
public async Task BufferBlock_PostWaitForBufferSpace()
{
var bufferBlock = new BufferBlock <ImportCustomer>(DataOptions);
var actionBlock = new ActionBlock <ImportCustomer>(importCustomer => PerformAction(importCustomer),
ExecutionOptions);
bufferBlock.LinkTo(actionBlock, LinkOptions);
var bulkService = new BulkCustomerDataService(100);
await foreach (var customer in bulkService.GetCustomersFromImportAsync())
{
// wait until block has capacity before sending more records
while (bufferBlock.Post(customer) == false)
{
await Task.Delay(10);
}
}
// ensure work completes prior to test exiting
bufferBlock.Complete();
await actionBlock.Completion;
Assert.AreEqual(100, _performActionCount, $"Action count is {_performActionCount}");
}
public async Task Test1()
{
var bulkCustomerService = new BulkCustomerDataService(100);
var customerService = new CustomerService();
await foreach (var importCustomer in bulkCustomerService.GetCustomersFromImportAsync())
{
await customerService.SaveCustomer(importCustomer.ToCustomer());
}
Assert.AreEqual(customerService.SavedCount, 100);
}
public async Task Test3()
{
var bulkCustomerService = new BulkCustomerDataService(10);
var customerService = new CustomerService();
await foreach (var importCustomer in bulkCustomerService.GetCustomersFromImportAsync())
{
customerService.SaveCustomer(importCustomer.ToCustomer());
}
Assert.IsTrue(customerService.SavedCount < 10);
}
public async Task Test2()
{
var bulkCustomerService = new BulkCustomerDataService(100);
var customerService = new CustomerService();
List <Task> tasks = new List <Task>();
await foreach (var importCustomer in bulkCustomerService.GetCustomersFromImportAsync())
{
tasks.Add(customerService.SaveCustomer(importCustomer.ToCustomer()));
}
Task.WaitAll(tasks.ToArray());
Assert.AreEqual(customerService.SavedCount, 100);
}
public async Task ActionBlock_Example()
{
var actionBlock = new ActionBlock <ImportCustomer>(importCustomer => PerformAction(importCustomer), ExecutionOptions);
var bulkService = new BulkCustomerDataService(10);
await foreach (var customer in bulkService.GetCustomersFromImportAsync())
{
await actionBlock.SendAsync(customer);
}
// ensure work completes prior to test exiting
actionBlock.Complete();
await actionBlock.Completion;
}
public async Task TransformManyBlock_Example()
{
var transformManyBlock = new TransformManyBlock <string, ImportCustomer>(JsonConvert.DeserializeObject <List <ImportCustomer> >, ExecutionOptions);
var actionBlock = new ActionBlock <ImportCustomer>(importCustomer => PerformAction(importCustomer), ExecutionOptions);
transformManyBlock.LinkTo(actionBlock, LinkOptions);
var bulkService = new BulkCustomerDataService(100);
var json = JsonConvert.SerializeObject(await bulkService.GetCustomersFromImportAsync().ToListAsync());
transformManyBlock.Post(json);
// ensure work completes prior to test exiting
transformManyBlock.Complete();
await actionBlock.Completion;
Assert.AreEqual(100, _performActionCount, $"Action count is {_performActionCount}");
}
public async Task Save_OneAtATime()
{
_currentTestCount = _customerCount / 1000; // too slow to do full count
var bulkDataService = new BulkCustomerDataService(_currentTestCount);
var saveCount = 0;
// this will load the entire list into memory
var importCustomers = bulkDataService.GetCustomersFromImportAsync();
await foreach (var importCustomer in importCustomers)
{
var customer = importCustomer.ToCustomer();
_customerService.SaveCustomer(customer).Wait();
saveCount++;
}
Assert.AreEqual(_currentTestCount, saveCount);
}
public async Task BatchBlock_Example()
{
var batchBlock = new BatchBlock <ImportCustomer>(10);
var actionBlock = new ActionBlock <ImportCustomer[]>(importCustomer => PerformAction(importCustomer), ExecutionOptions);
batchBlock.LinkTo(actionBlock, LinkOptions);
var bulkService = new BulkCustomerDataService(100);
await foreach (var importCustomer in bulkService.GetCustomersFromImportAsync())
{
await batchBlock.SendAsync(importCustomer);
}
// ensure work completes before exiting test
batchBlock.Complete();
await actionBlock.Completion;
Assert.AreEqual(10, _performActionCount, $"Action count is {_performActionCount}");
}
public async Task TransformBlock_Example()
{
var transformBlock = new TransformBlock <ImportCustomer, Customer>(importCustomer => importCustomer.ToCustomer(), ExecutionOptions);
var actionBlock = new ActionBlock <Customer>(customer => PerformAction(customer), ExecutionOptions);
transformBlock.LinkTo(actionBlock, LinkOptions);
var bulkService = new BulkCustomerDataService(100);
await foreach (var importCustomer in bulkService.GetCustomersFromImportAsync())
{
await transformBlock.SendAsync(importCustomer);
}
// ensure work completes prior to test exiting
transformBlock.Complete();
await actionBlock.Completion;
Assert.AreEqual(100, _performActionCount, $"Action count is {_performActionCount}");
}
public async Task BufferBlock_PostRejectsMessageOnceFull()
{
var bufferBlock = new BufferBlock <ImportCustomer>(DataOptions);
var actionBlock = new ActionBlock <ImportCustomer>(importCustomer => PerformAction(importCustomer), ExecutionOptions);
bufferBlock.LinkTo(actionBlock, LinkOptions);
var bulkService = new BulkCustomerDataService(100);
await foreach (var customer in bulkService.GetCustomersFromImportAsync())
{
// this will fail once the block reaches capacity
bufferBlock.Post(customer);
}
// ensure work completes prior to test exiting
bufferBlock.Complete();
await actionBlock.Completion;
// not all actions complete because post rejected records
Assert.IsTrue(_performActionCount < 100, $"Action count is {_performActionCount}");
}
public async Task BufferBlock_SendAsyncWillWait()
{
var bufferBlock = new BufferBlock <ImportCustomer>(DataOptions);
var actionBlock = new ActionBlock <ImportCustomer>(importCustomer => PerformAction(importCustomer),
ExecutionOptions);
bufferBlock.LinkTo(actionBlock, LinkOptions);
var bulkService = new BulkCustomerDataService(100);
await foreach (var customer in bulkService.GetCustomersFromImportAsync())
{
// send async will return an incomplete task when block is full that we can wait on
await bufferBlock.SendAsync(customer);
}
// ensure work completes prior to test exiting
bufferBlock.Complete();
await actionBlock.Completion;
Assert.AreEqual(100, _performActionCount, $"Action count is {_performActionCount}");
}
public async Task BroadcastBlock_Example()
{
var broadcastBlock = new BroadcastBlock <ImportCustomer>(null, new DataflowBlockOptions());
var actionBlock1 = new ActionBlock <ImportCustomer>(importCustomer => PerformAction(importCustomer, "Action-1", 50), ExecutionOptions);
var actionBlock2 = new ActionBlock <ImportCustomer>(importCustomer => PerformAction(importCustomer, "Action-2", 100), ExecutionOptions);
broadcastBlock.LinkTo(actionBlock1, LinkOptions);
broadcastBlock.LinkTo(actionBlock2, LinkOptions);
var bulkService = new BulkCustomerDataService(100);
await foreach (var customer in bulkService.GetCustomersFromImportAsync())
{
await broadcastBlock.SendAsync(customer);
}
// ensure work completes prior to test exiting
broadcastBlock.Complete();
Task.WaitAll(actionBlock1.Completion, actionBlock2.Completion);
// each action block received every message
Assert.AreEqual(200, _performActionCount, $"Action count is {_performActionCount}");
}
public async Task Save_Batch_Parallel()
{
_currentTestCount = _customerCount;
var batch = new ConcurrentQueue <Customer>();
var semaphore = new SemaphoreSlim(1, 1);
var bulkDataService = new BulkCustomerDataService(_currentTestCount);
var saveCount = 0;
// this will load the entire list into memory
var importCustomers = await bulkDataService.GetCustomersFromImportAsync().ToListAsync();
Parallel.ForEach(importCustomers,
new ParallelOptions()
{
MaxDegreeOfParallelism = 10
}, importCustomer =>
{
var customer = importCustomer.ToCustomer();
batch.Enqueue(customer);
if (batch.Count >= 100)
{
var customersToSave = new List <Customer>();
// ensure only 1 thread is sucking records from queue at a time
semaphore.Wait();
if (batch.Count >= 100)
{
for (int i = 0; i < 100; i++)
{
if (batch.TryDequeue(out var batchCustomer))
{
customersToSave.Add(batchCustomer);
}
}
