/// <summary>
/// Send an enumerable of message objects to a given topic.
/// </summary>
/// <param name="topic">The name of the kafka topic to send the messages to.</param>
/// <param name="messages">The enumerable of messages that will be sent to the given topic.</param>
/// <param name="acks">The required level of acknowlegment from the kafka server. 0=none, 1=writen to leader, 2+=writen to replicas, -1=writen to all replicas.</param>
/// <param name="timeout">Interal kafka timeout to wait for the requested level of ack to occur before returning. Defaults to 1000ms.</param>
/// <param name="codec">The codec to apply to the message collection. Defaults to none.</param>
/// <returns>List of ProduceResponses from each partition sent to or empty list if acks = 0.</returns>
public async Task <List <ProduceResponse> > SendMessageAsync(string topic, IEnumerable <Message> messages, Int16 acks = 1,
TimeSpan?timeout = null, MessageCodec codec = MessageCodec.CodecNone)
{
if (_stopToken.IsCancellationRequested)
{
throw new ObjectDisposedException("Cannot send new documents as producer is disposing.");
}
if (timeout == null)
{
timeout = TimeSpan.FromMilliseconds(DefaultAckTimeoutMS);
}
var batch = messages.Select(message => new TopicMessage
{
Acks = acks,
Codec = codec,
Timeout = timeout.Value,
Topic = topic,
Message = message
}).ToList();
await _nagleBlockingCollection.AddRangeAsync(batch, _stopToken.Token).ConfigureAwait(false);
var results = new List <ProduceResponse>();
foreach (var topicMessage in batch)
{
results.Add(await topicMessage.Tcs.Task.ConfigureAwait(false));
}
return(results.Distinct().ToList());
}
public async void CollectionAddRangeAsyncShouldBeAbleToCancel()
{
var collection = new NagleBlockingCollection<int>(1);
using (var cancellationToken = new CancellationTokenSource())
{
var cancelledTask = collection.AddRangeAsync(Enumerable.Range(0, 5), cancellationToken.Token);
cancellationToken.Cancel();
await cancelledTask;
}
}
public async void CollectionAddRangeAsyncShouldBeAbleToCancel()
{
var collection = new NagleBlockingCollection <int>(1);
using (var cancellationToken = new CancellationTokenSource())
{
var cancelledTask = collection.AddRangeAsync(Enumerable.Range(0, 5), cancellationToken.Token);
cancellationToken.Cancel();
await cancelledTask;
}
}
public async void TakeBatchShouldRemoveItemsFromCollection()
{
const int expectedCount = 10;
var collection = new NagleBlockingCollection <int>(100);
await collection.AddRangeAsync(Enumerable.Range(0, expectedCount), CancellationToken.None);
var data = await collection.TakeBatch(expectedCount, TimeSpan.FromMilliseconds(100), CancellationToken.None);
Assert.That(data.Count, Is.EqualTo(expectedCount));
Assert.That(collection.Count, Is.EqualTo(0));
}
public async void TakeAsyncShouldReturnAsSoonAsBatchSizeArrived()
{
var collection = new NagleBlockingCollection <int>(100);
var dataTask = collection.TakeBatch(10, TimeSpan.FromSeconds(5), CancellationToken.None);
collection.AddRangeAsync(Enumerable.Range(0, 10), CancellationToken.None);
await dataTask;
Assert.That(collection.Count, Is.EqualTo(0));
}
public async void CollectionShouldWaitXForBatchSizeToCollect()
{
const int expectedDelay = 100;
const int expectedCount = 10;
var collection = new NagleBlockingCollection <int>(100);
await collection.AddRangeAsync(Enumerable.Range(0, expectedCount), CancellationToken.None);
var sw = Stopwatch.StartNew();
var data = await collection.TakeBatch(expectedCount + 1, TimeSpan.FromMilliseconds(expectedDelay), CancellationToken.None);
Assert.That(sw.ElapsedMilliseconds, Is.GreaterThanOrEqualTo(expectedDelay));
Assert.That(data.Count, Is.EqualTo(expectedCount));
}
public async void CollectionShouldReportCorrectBufferCount()
{
var collection = new NagleBlockingCollection <int>(100);
var dataTask = collection.TakeBatch(10, TimeSpan.FromSeconds(5), CancellationToken.None);
await collection.AddRangeAsync(Enumerable.Range(0, 9), CancellationToken.None);
Assert.That(collection.Count, Is.EqualTo(9));
collection.AddAsync(1, CancellationToken.None);
var data = await dataTask;
Assert.That(data.Count, Is.EqualTo(10));
Assert.That(collection.Count, Is.EqualTo(0));
}
public async void EnsureCollectionBlocksAtCapacity()
{
const int blockingCount = 10;
const int expectedCount = 5;
var collection = new NagleBlockingCollection <int>(blockingCount);
var addTask = Task.Factory.StartNew(() => collection.AddRangeAsync(Enumerable.Range(0, blockingCount + expectedCount),
CancellationToken.None).Wait());
TaskTest.WaitFor(() => collection.Count >= blockingCount);
Assert.That(collection.Count, Is.EqualTo(blockingCount), "The collection should only contain 10 items.");
Assert.That(addTask.Status, Is.EqualTo(TaskStatus.Running), "The task should be blocking.");
//unblock the collection
await collection.TakeBatch(blockingCount, TimeSpan.FromMilliseconds(100), CancellationToken.None);
await addTask;
Assert.That(collection.Count, Is.EqualTo(expectedCount));
Assert.That(addTask.Status, Is.EqualTo(TaskStatus.RanToCompletion));
}
public async void EnsureCollectionBlocksAtCapacity()
{
const int blockingCount = 10;
const int expectedCount = 5;
var collection = new NagleBlockingCollection<int>(blockingCount);
var addTask = Task.Factory.StartNew(() => collection.AddRangeAsync(Enumerable.Range(0, blockingCount + expectedCount),
CancellationToken.None).Wait());
TaskTest.WaitFor(() => collection.Count >= blockingCount);
Assert.That(collection.Count, Is.EqualTo(blockingCount), "The collection should only contain 10 items.");
Assert.That(addTask.Status, Is.EqualTo(TaskStatus.Running), "The task should be blocking.");
//unblock the collection
await collection.TakeBatch(blockingCount, TimeSpan.FromMilliseconds(100), CancellationToken.None);
await addTask;
Assert.That(collection.Count, Is.EqualTo(expectedCount));
Assert.That(addTask.Status, Is.EqualTo(TaskStatus.RanToCompletion));
}
public async void TakeBatchShouldRemoveItemsFromCollection()
{
const int expectedCount = 10;
var collection = new NagleBlockingCollection<int>(100);
await collection.AddRangeAsync(Enumerable.Range(0, expectedCount), CancellationToken.None);
var data = await collection.TakeBatch(expectedCount, TimeSpan.FromMilliseconds(100), CancellationToken.None);
Assert.That(data.Count, Is.EqualTo(expectedCount));
Assert.That(collection.Count, Is.EqualTo(0));
}
public async void TakeAsyncShouldReturnAsSoonAsBatchSizeArrived()
{
var collection = new NagleBlockingCollection<int>(100);
var dataTask = collection.TakeBatch(10, TimeSpan.FromSeconds(5), CancellationToken.None);
collection.AddRangeAsync(Enumerable.Range(0, 10), CancellationToken.None);
await dataTask;
Assert.That(collection.Count, Is.EqualTo(0));
}
public async void CollectionShouldReportCorrectBufferCount()
{
var collection = new NagleBlockingCollection<int>(100);
var dataTask = collection.TakeBatch(10, TimeSpan.FromSeconds(5), CancellationToken.None);
await collection.AddRangeAsync(Enumerable.Range(0, 9), CancellationToken.None);
Assert.That(collection.Count, Is.EqualTo(9));
collection.AddAsync(1, CancellationToken.None);
var data = await dataTask;
Assert.That(data.Count, Is.EqualTo(10));
Assert.That(collection.Count, Is.EqualTo(0));
}
public async void CollectionShouldWaitXForBatchSizeToCollect()
{
const int expectedDelay = 100;
const int expectedCount = 10;
var collection = new NagleBlockingCollection<int>(100);
await collection.AddRangeAsync(Enumerable.Range(0, expectedCount), CancellationToken.None);
var sw = Stopwatch.StartNew();
var data = await collection.TakeBatch(expectedCount + 1, TimeSpan.FromMilliseconds(expectedDelay), CancellationToken.None);
Assert.That(sw.ElapsedMilliseconds, Is.GreaterThanOrEqualTo(expectedDelay));
Assert.That(data.Count, Is.EqualTo(expectedCount));
}
