/// <summary>
/// Send an enumerable of message objects to a given topic.
/// </summary>
/// <param name="topic">The name of the kafka topic to send the messages to.</param>
/// <param name="messages">The enumerable of messages that will be sent to the given topic.</param>
/// <param name="acks">The required level of acknowlegment from the kafka server. 0=none, 1=writen to leader, 2+=writen to replicas, -1=writen to all replicas.</param>
/// <param name="timeout">Interal kafka timeout to wait for the requested level of ack to occur before returning. Defaults to 1000ms.</param>
/// <param name="codec">The codec to apply to the message collection. Defaults to none.</param>
/// <returns>List of ProduceResponses from each partition sent to or empty list if acks = 0.</returns>
public async Task <List <ProduceResponse> > SendMessageAsync(string topic, IEnumerable <Message> messages, Int16 acks = 1,
TimeSpan?timeout = null, MessageCodec codec = MessageCodec.CodecNone)
{
if (_stopToken.IsCancellationRequested)
{
throw new ObjectDisposedException("Cannot send new documents as producer is disposing.");
}
if (timeout == null)
{
timeout = TimeSpan.FromMilliseconds(DefaultAckTimeoutMS);
}
var batch = messages.Select(message => new TopicMessage
{
Acks = acks,
Codec = codec,
Timeout = timeout.Value,
Topic = topic,
Message = message
}).ToList();
_asyncCollection.AddRange(batch);
await Task.WhenAll(batch.Select(x => x.Tcs.Task));
return(batch.Select(topicMessage => topicMessage.Tcs.Task.Result)
.Distinct()
.ToList());
}
public void AddRangeShouldBePerformant()
{
AsyncCollection <int> collection = new AsyncCollection <int>();
var sw = Stopwatch.StartNew();
collection.AddRange(Enumerable.Range(0, 1000000));
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}
public async void TakeAsyncShouldReturnAsSoonAsBatchSizeArrived()
{
var collection = new AsyncCollection <int>();
var dataTask = collection.TakeAsync(10, TimeSpan.FromSeconds(5), CancellationToken.None);
collection.AddRange(Enumerable.Range(0, 10));
await dataTask;
Assert.That(collection.Count, Is.EqualTo(0));
}
public async void TakeAsyncShouldRemoveItemsFromCollection()
{
const int expectedCount = 10;
var collection = new AsyncCollection <int>();
collection.AddRange(Enumerable.Range(0, expectedCount));
var data = await collection.TakeAsync(expectedCount, TimeSpan.FromMilliseconds(100), CancellationToken.None);
Assert.That(data.Count, Is.EqualTo(expectedCount));
Assert.That(collection.Count, Is.EqualTo(0));
}
public async Task CollectionShouldBlockOnMaxBuffer()
{
var collection = new AsyncCollection <int>();
var task = Task.Factory.StartNew(() => collection.AddRange(Enumerable.Range(0, 10)));
await TaskTest.WaitFor(() => collection.Count >= 9);
Assert.That(collection.Count, Is.EqualTo(9), "Buffer should block at 9 items.");
Assert.That(task.IsCompleted, Is.False, "Task should be blocking on last item.");
var item = collection.Pop();
await TaskTest.WaitFor(() => task.IsCompleted);
Assert.That(task.IsCompleted, Is.True, "Task should complete after room is made in buffer.");
Assert.That(collection.Count, Is.EqualTo(9), "There should now be 9 items in the buffer.");
}
public async void CollectionShouldReportCorrectBufferCount()
{
var collection = new AsyncCollection<int>();
var dataTask = collection.TakeAsync(10, TimeSpan.FromHours(5), CancellationToken.None);
collection.AddRange(Enumerable.Range(0, 9));
Assert.That(collection.Count, Is.EqualTo(9));
collection.Add(1);
var data = await dataTask;
Assert.That(data.Count, Is.EqualTo(10));
Assert.That(collection.Count, Is.EqualTo(0));
}
public async void TakeAsyncShouldBePerformant()
{
const int dataSize = 1000000;
AsyncCollection <int> collection = new AsyncCollection <int>();
collection.AddRange(Enumerable.Range(0, dataSize));
var sw = Stopwatch.StartNew();
var list = await collection.TakeAsync(dataSize, TimeSpan.FromSeconds(1), CancellationToken.None);
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(list.Count, Is.EqualTo(dataSize));
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}
public async void TakeAsyncShouldWaitXForBatchSizeToCollect()
{
const int expectedDelay = 100;
const int expectedCount = 10;
var collection = new AsyncCollection <int>();
collection.AddRange(Enumerable.Range(0, expectedCount));
var sw = Stopwatch.StartNew();
var data = await collection.TakeAsync(expectedCount + 1, TimeSpan.FromMilliseconds(expectedDelay), CancellationToken.None);
Assert.That(sw.ElapsedMilliseconds, Is.GreaterThanOrEqualTo(expectedDelay));
Assert.That(data.Count, Is.EqualTo(expectedCount));
}
public async void CollectionShouldReportCorrectBufferCount()
{
var collection = new AsyncCollection <int>();
var dataTask = collection.TakeAsync(10, TimeSpan.FromHours(5), CancellationToken.None);
collection.AddRange(Enumerable.Range(0, 9));
Assert.That(collection.Count, Is.EqualTo(9));
collection.Add(1);
var data = await dataTask;
Assert.That(data.Count, Is.EqualTo(10));
Assert.That(collection.Count, Is.EqualTo(0));
}
public void AddAndRemoveShouldBePerformant()
{
const int dataSize = 1000000;
var collection = new AsyncCollection <int>();
var sw = Stopwatch.StartNew();
var receivedData = new List <int>();
Parallel.Invoke(
() => collection.AddRange(Enumerable.Range(0, dataSize)),
() => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result);
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(receivedData.Count, Is.EqualTo(dataSize));
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}
public async Task AddAndRemoveShouldBePerformant()
{
AsyncCollection <int> collection = new AsyncCollection <int>();
const int dataSize = 1000000;
List <int> receivedData = null;// new List<int>(dataSize);
var sw = Stopwatch.StartNew();
var t = Task.Run(() => collection.AddRange(Enumerable.Range(0, dataSize)));
var t2 = Task.Run(() => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result);
await Task.WhenAll(t, t2);
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(receivedData.Count, Is.EqualTo(dataSize));
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}
public void CollectionShouldBlockOnMaxBuffer()
{
var collection = new AsyncCollection<int>();
var task = Task.Factory.StartNew(() => collection.AddRange(Enumerable.Range(0, 10)));
TaskTest.WaitFor(() => collection.Count >= 9);
Assert.That(collection.Count, Is.EqualTo(9), "Buffer should block at 9 items.");
Assert.That(task.IsCompleted, Is.False, "Task should be blocking on last item.");
var item = collection.Pop();
TaskTest.WaitFor(() => task.IsCompleted);
Assert.That(task.IsCompleted, Is.True, "Task should complete after room is made in buffer.");
Assert.That(collection.Count, Is.EqualTo(9), "There should now be 9 items in the buffer.");
}
public async void TakeAsyncShouldBePerformant()
{
const int dataSize = 1000000;
var collection = new AsyncCollection<int>();
collection.AddRange(Enumerable.Range(0, dataSize));
var sw = Stopwatch.StartNew();
var list = await collection.TakeAsync(dataSize, TimeSpan.FromSeconds(1), CancellationToken.None);
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(list.Count, Is.EqualTo(dataSize));
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}
public void AddAndRemoveShouldBePerformant()
{
const int dataSize = 1000000;
var collection = new AsyncCollection<int>();
var sw = Stopwatch.StartNew();
var receivedData = new List<int>();
Parallel.Invoke(
() => collection.AddRange(Enumerable.Range(0, dataSize)),
() => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result);
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(receivedData.Count, Is.EqualTo(dataSize));
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}
public void AddRangeShouldBePerformant()
{
var sw = Stopwatch.StartNew();
var collection = new AsyncCollection<int>();
collection.AddRange(Enumerable.Range(0, 1000000));
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}
public async void TakeAsyncShouldReturnAsSoonAsBatchSizeArrived()
{
var collection = new AsyncCollection<int>();
var dataTask = collection.TakeAsync(10, TimeSpan.FromSeconds(5), CancellationToken.None);
collection.AddRange(Enumerable.Range(0, 10));
await dataTask;
Assert.That(collection.Count, Is.EqualTo(0));
}
public async void TakeAsyncShouldWaitXForBatchSizeToCollect()
{
const int expectedDelay = 100;
const int expectedCount = 10;
var collection = new AsyncCollection<int>();
collection.AddRange(Enumerable.Range(0, expectedCount));
var sw = Stopwatch.StartNew();
var data = await collection.TakeAsync(expectedCount + 1, TimeSpan.FromMilliseconds(expectedDelay), CancellationToken.None);
Assert.That(sw.ElapsedMilliseconds, Is.GreaterThanOrEqualTo(expectedDelay));
Assert.That(data.Count, Is.EqualTo(expectedCount));
}
public async void TakeAsyncShouldRemoveItemsFromCollection()
{
const int expectedCount = 10;
var collection = new AsyncCollection<int>();
collection.AddRange(Enumerable.Range(0, expectedCount));
var data = await collection.TakeAsync(expectedCount, TimeSpan.FromMilliseconds(100), CancellationToken.None);
Assert.That(data.Count, Is.EqualTo(expectedCount));
Assert.That(collection.Count, Is.EqualTo(0));
}
public async Task AddAndRemoveShouldBePerformant()
{
AsyncCollection<int> collection = new AsyncCollection<int>();
const int dataSize = 1000000;
List<int> receivedData = null;// new List<int>(dataSize);
var sw = Stopwatch.StartNew();
var t = Task.Run(() => collection.AddRange(Enumerable.Range(0, dataSize)));
var t2 = Task.Run(() => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result);
await Task.WhenAll(t, t2);
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(receivedData.Count, Is.EqualTo(dataSize));
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}