/// <summary> /// Send an enumerable of message objects to a given topic. /// </summary> /// <param name="topic">The name of the kafka topic to send the messages to.</param> /// <param name="messages">The enumerable of messages that will be sent to the given topic.</param> /// <param name="acks">The required level of acknowlegment from the kafka server. 0=none, 1=writen to leader, 2+=writen to replicas, -1=writen to all replicas.</param> /// <param name="timeout">Interal kafka timeout to wait for the requested level of ack to occur before returning. Defaults to 1000ms.</param> /// <param name="codec">The codec to apply to the message collection. Defaults to none.</param> /// <returns>List of ProduceResponses from each partition sent to or empty list if acks = 0.</returns> public async Task <List <ProduceResponse> > SendMessageAsync(string topic, IEnumerable <Message> messages, Int16 acks = 1, TimeSpan?timeout = null, MessageCodec codec = MessageCodec.CodecNone) { if (_stopToken.IsCancellationRequested) { throw new ObjectDisposedException("Cannot send new documents as producer is disposing."); } if (timeout == null) { timeout = TimeSpan.FromMilliseconds(DefaultAckTimeoutMS); } var batch = messages.Select(message => new TopicMessage { Acks = acks, Codec = codec, Timeout = timeout.Value, Topic = topic, Message = message }).ToList(); _asyncCollection.AddRange(batch); await Task.WhenAll(batch.Select(x => x.Tcs.Task)); return(batch.Select(topicMessage => topicMessage.Tcs.Task.Result) .Distinct() .ToList()); }
public void AddRangeShouldBePerformant() { AsyncCollection <int> collection = new AsyncCollection <int>(); var sw = Stopwatch.StartNew(); collection.AddRange(Enumerable.Range(0, 1000000)); sw.Stop(); Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds); Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200)); }
public async void TakeAsyncShouldReturnAsSoonAsBatchSizeArrived() { var collection = new AsyncCollection <int>(); var dataTask = collection.TakeAsync(10, TimeSpan.FromSeconds(5), CancellationToken.None); collection.AddRange(Enumerable.Range(0, 10)); await dataTask; Assert.That(collection.Count, Is.EqualTo(0)); }
public async void TakeAsyncShouldRemoveItemsFromCollection() { const int expectedCount = 10; var collection = new AsyncCollection <int>(); collection.AddRange(Enumerable.Range(0, expectedCount)); var data = await collection.TakeAsync(expectedCount, TimeSpan.FromMilliseconds(100), CancellationToken.None); Assert.That(data.Count, Is.EqualTo(expectedCount)); Assert.That(collection.Count, Is.EqualTo(0)); }
public async Task CollectionShouldBlockOnMaxBuffer() { var collection = new AsyncCollection <int>(); var task = Task.Factory.StartNew(() => collection.AddRange(Enumerable.Range(0, 10))); await TaskTest.WaitFor(() => collection.Count >= 9); Assert.That(collection.Count, Is.EqualTo(9), "Buffer should block at 9 items."); Assert.That(task.IsCompleted, Is.False, "Task should be blocking on last item."); var item = collection.Pop(); await TaskTest.WaitFor(() => task.IsCompleted); Assert.That(task.IsCompleted, Is.True, "Task should complete after room is made in buffer."); Assert.That(collection.Count, Is.EqualTo(9), "There should now be 9 items in the buffer."); }
public async void CollectionShouldReportCorrectBufferCount() { var collection = new AsyncCollection<int>(); var dataTask = collection.TakeAsync(10, TimeSpan.FromHours(5), CancellationToken.None); collection.AddRange(Enumerable.Range(0, 9)); Assert.That(collection.Count, Is.EqualTo(9)); collection.Add(1); var data = await dataTask; Assert.That(data.Count, Is.EqualTo(10)); Assert.That(collection.Count, Is.EqualTo(0)); }
public async void TakeAsyncShouldBePerformant() { const int dataSize = 1000000; AsyncCollection <int> collection = new AsyncCollection <int>(); collection.AddRange(Enumerable.Range(0, dataSize)); var sw = Stopwatch.StartNew(); var list = await collection.TakeAsync(dataSize, TimeSpan.FromSeconds(1), CancellationToken.None); sw.Stop(); Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds); Assert.That(list.Count, Is.EqualTo(dataSize)); Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200)); }
public async void TakeAsyncShouldWaitXForBatchSizeToCollect() { const int expectedDelay = 100; const int expectedCount = 10; var collection = new AsyncCollection <int>(); collection.AddRange(Enumerable.Range(0, expectedCount)); var sw = Stopwatch.StartNew(); var data = await collection.TakeAsync(expectedCount + 1, TimeSpan.FromMilliseconds(expectedDelay), CancellationToken.None); Assert.That(sw.ElapsedMilliseconds, Is.GreaterThanOrEqualTo(expectedDelay)); Assert.That(data.Count, Is.EqualTo(expectedCount)); }
public async void CollectionShouldReportCorrectBufferCount() { var collection = new AsyncCollection <int>(); var dataTask = collection.TakeAsync(10, TimeSpan.FromHours(5), CancellationToken.None); collection.AddRange(Enumerable.Range(0, 9)); Assert.That(collection.Count, Is.EqualTo(9)); collection.Add(1); var data = await dataTask; Assert.That(data.Count, Is.EqualTo(10)); Assert.That(collection.Count, Is.EqualTo(0)); }
public void AddAndRemoveShouldBePerformant() { const int dataSize = 1000000; var collection = new AsyncCollection <int>(); var sw = Stopwatch.StartNew(); var receivedData = new List <int>(); Parallel.Invoke( () => collection.AddRange(Enumerable.Range(0, dataSize)), () => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result); sw.Stop(); Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds); Assert.That(receivedData.Count, Is.EqualTo(dataSize)); Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200)); }
public async Task AddAndRemoveShouldBePerformant() { AsyncCollection <int> collection = new AsyncCollection <int>(); const int dataSize = 1000000; List <int> receivedData = null;// new List<int>(dataSize); var sw = Stopwatch.StartNew(); var t = Task.Run(() => collection.AddRange(Enumerable.Range(0, dataSize))); var t2 = Task.Run(() => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result); await Task.WhenAll(t, t2); sw.Stop(); Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds); Assert.That(receivedData.Count, Is.EqualTo(dataSize)); Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200)); }
public void CollectionShouldBlockOnMaxBuffer() { var collection = new AsyncCollection<int>(); var task = Task.Factory.StartNew(() => collection.AddRange(Enumerable.Range(0, 10))); TaskTest.WaitFor(() => collection.Count >= 9); Assert.That(collection.Count, Is.EqualTo(9), "Buffer should block at 9 items."); Assert.That(task.IsCompleted, Is.False, "Task should be blocking on last item."); var item = collection.Pop(); TaskTest.WaitFor(() => task.IsCompleted); Assert.That(task.IsCompleted, Is.True, "Task should complete after room is made in buffer."); Assert.That(collection.Count, Is.EqualTo(9), "There should now be 9 items in the buffer."); }
public async void TakeAsyncShouldBePerformant() { const int dataSize = 1000000; var collection = new AsyncCollection<int>(); collection.AddRange(Enumerable.Range(0, dataSize)); var sw = Stopwatch.StartNew(); var list = await collection.TakeAsync(dataSize, TimeSpan.FromSeconds(1), CancellationToken.None); sw.Stop(); Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds); Assert.That(list.Count, Is.EqualTo(dataSize)); Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200)); }
public void AddAndRemoveShouldBePerformant() { const int dataSize = 1000000; var collection = new AsyncCollection<int>(); var sw = Stopwatch.StartNew(); var receivedData = new List<int>(); Parallel.Invoke( () => collection.AddRange(Enumerable.Range(0, dataSize)), () => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result); sw.Stop(); Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds); Assert.That(receivedData.Count, Is.EqualTo(dataSize)); Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200)); }
public void AddRangeShouldBePerformant() { var sw = Stopwatch.StartNew(); var collection = new AsyncCollection<int>(); collection.AddRange(Enumerable.Range(0, 1000000)); sw.Stop(); Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds); Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200)); }
public async void TakeAsyncShouldReturnAsSoonAsBatchSizeArrived() { var collection = new AsyncCollection<int>(); var dataTask = collection.TakeAsync(10, TimeSpan.FromSeconds(5), CancellationToken.None); collection.AddRange(Enumerable.Range(0, 10)); await dataTask; Assert.That(collection.Count, Is.EqualTo(0)); }
public async void TakeAsyncShouldWaitXForBatchSizeToCollect() { const int expectedDelay = 100; const int expectedCount = 10; var collection = new AsyncCollection<int>(); collection.AddRange(Enumerable.Range(0, expectedCount)); var sw = Stopwatch.StartNew(); var data = await collection.TakeAsync(expectedCount + 1, TimeSpan.FromMilliseconds(expectedDelay), CancellationToken.None); Assert.That(sw.ElapsedMilliseconds, Is.GreaterThanOrEqualTo(expectedDelay)); Assert.That(data.Count, Is.EqualTo(expectedCount)); }
public async void TakeAsyncShouldRemoveItemsFromCollection() { const int expectedCount = 10; var collection = new AsyncCollection<int>(); collection.AddRange(Enumerable.Range(0, expectedCount)); var data = await collection.TakeAsync(expectedCount, TimeSpan.FromMilliseconds(100), CancellationToken.None); Assert.That(data.Count, Is.EqualTo(expectedCount)); Assert.That(collection.Count, Is.EqualTo(0)); }
public async Task AddAndRemoveShouldBePerformant() { AsyncCollection<int> collection = new AsyncCollection<int>(); const int dataSize = 1000000; List<int> receivedData = null;// new List<int>(dataSize); var sw = Stopwatch.StartNew(); var t = Task.Run(() => collection.AddRange(Enumerable.Range(0, dataSize))); var t2 = Task.Run(() => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result); await Task.WhenAll(t, t2); sw.Stop(); Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds); Assert.That(receivedData.Count, Is.EqualTo(dataSize)); Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200)); }