public async void ListenerRecoveryTest()
{
kafka4net.Tracing.EtwTrace.Marker("ListenerRecoveryTest");
const int count = 10000;
var topic = "part33." + _rnd.Next();
VagrantBrokerUtil.CreateTopic(topic, 6, 3);
var producer = new Producer(_seed2Addresses, new ProducerConfiguration(topic));
_log.Debug("Connecting");
await producer.ConnectAsync();
_log.Debug("Filling out {0} with {1} messages", topic, count);
var sentList = await Enumerable.Range(0, count)
.Select(i => new Message { Value = BitConverter.GetBytes(i) })
.ToObservable()
.Do(producer.Send)
.Select(msg => BitConverter.ToInt32(msg.Value, 0))
.ToList();
await Task.Delay(TimeSpan.FromSeconds(1));
var heads = await producer.Cluster.FetchPartitionOffsetsAsync(topic, ConsumerLocation.TopicStart);
var tails = await producer.Cluster.FetchPartitionOffsetsAsync(topic, ConsumerLocation.TopicEnd);
_log.Info("Done sending messages. Closing producer.");
await producer.CloseAsync(TimeSpan.FromSeconds(5));
_log.Info("Producer closed, starting consumer subscription.");
var messagesInTopic = (int)tails.MessagesSince(heads);
_log.Info("Topic offsets indicate producer sent {0} messages.", messagesInTopic);
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, new StartPositionTopicStart(), maxBytesPerFetch: 4 * 8));
var current = 0;
var received = new ReplaySubject<ReceivedMessage>();
Task stopBrokerTask = null;
var consumerSubscription = consumer.OnMessageArrived.
Subscribe(async msg =>
{
current++;
if (current == 18)
{
stopBrokerTask = Task.Factory.StartNew(() => VagrantBrokerUtil.StopBrokerLeaderForPartition(consumer.Cluster, consumer.Topic, msg.Partition), CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default);
}
received.OnNext(msg);
//_log.Info("Got: {0}", BitConverter.ToInt32(msg.Value, 0));
});
await consumer.IsConnected;
_log.Info("Waiting for receiver complete");
var receivedList = await received.Select(msg => BitConverter.ToInt32(msg.Value, 0)).Take(messagesInTopic).
TakeUntil(DateTime.Now.AddSeconds(60)).ToList().ToTask();
if (stopBrokerTask != null)
await stopBrokerTask.TimeoutAfter(TimeSpan.FromSeconds(10));
tails = await consumer.Cluster.FetchPartitionOffsetsAsync(topic, ConsumerLocation.TopicEnd);
_log.Info("Receiver complete. Disposing Subscription");
consumerSubscription.Dispose();
_log.Info("Consumer subscription disposed. Closing consumer.");
consumer.Dispose();
_log.Info("Consumer closed.");
_log.Info("Sum of offsets: {0}", tails.MessagesSince(heads));
_log.Info("Offsets: [{0}]", string.Join(",", tails.Partitions.Select(p => string.Format("{0}:{1}", p, tails.NextOffset(p)))));
if (messagesInTopic != receivedList.Count)
{
// log some debug info.
_log.Error("Did not receive all messages. Messages sent but NOT received: {0}", string.Join(",", sentList.Except(receivedList).OrderBy(i => i)));
}
Assert.AreEqual(messagesInTopic, receivedList.Count);
kafka4net.Tracing.EtwTrace.Marker("/ListenerRecoveryTest");
}
public async void LeaderDownProducerAndConsumerRecovery()
{
kafka4net.Tracing.EtwTrace.Marker("LeaderDownProducerAndConsumerRecovery");
string topic = "part32." + _rnd.Next();
VagrantBrokerUtil.CreateTopic(topic, 3, 2);
var sent = new List<string>();
var confirmedSent1 = new List<string>();
var producer = new Producer(_seed2Addresses, new ProducerConfiguration(topic));
producer.OnSuccess += msgs =>
{
msgs.ForEach(msg => confirmedSent1.Add(Encoding.UTF8.GetString(msg.Value)));
_log.Debug("Sent {0} messages", msgs.Length);
};
await producer.ConnectAsync();
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, new StartPositionTopicEnd()));
const int postCount = 100;
const int postCount2 = 50;
//
// Read messages
//
var received = new List<ReceivedMessage>();
var receivedEvents = new ReplaySubject<ReceivedMessage>();
var consumerSubscription = consumer.OnMessageArrived.
Synchronize().
Subscribe(msg =>
{
received.Add(msg);
receivedEvents.OnNext(msg);
_log.Debug("Received {0}/{1}", Encoding.UTF8.GetString(msg.Value), received.Count);
});
await consumer.IsConnected;
//
// Send #1
//
_log.Info("Start sender");
Observable.Interval(TimeSpan.FromMilliseconds(200)).
Take(postCount).
Subscribe(
i => {
var msg = "msg " + i;
producer.Send(new Message { Value = Encoding.UTF8.GetBytes(msg) });
sent.Add("msg " + i);
},
() => _log.Info("Producer complete")
);
// wait for first 50 messages to arrive
_log.Info("Waiting for first {0} messages to arrive", postCount2);
await receivedEvents.Take(postCount2).Count().ToTask();
Assert.AreEqual(postCount2, received.Count);
_log.Info("Stopping broker");
var stoppedBroker = VagrantBrokerUtil.StopBrokerLeaderForPartition(producer.Cluster, topic, 0);
_log.Debug("Stopped broker {0}", stoppedBroker);
// post another 50 messages
_log.Info("Sending another {0} messages", postCount2);
var sender2 = Observable.Interval(TimeSpan.FromMilliseconds(200)).
Take(postCount2).
Publish().RefCount();
//
// Send #2
//
sender2.Subscribe(
i => {
var msg = "msg #2 " + i;
producer.Send(new Message { Value = Encoding.UTF8.GetBytes(msg) });
sent.Add(msg);
_log.Debug("Sent msg #2 {0}", i);
},
() => _log.Info("Producer #2 complete")
);
_log.Info("Waiting for #2 sender to complete");
await sender2.ToTask();
_log.Info("Waiting for producer.Close");
await producer.CloseAsync(TimeSpan.FromSeconds(60));
_log.Info("Waiting 4sec for remaining messages");
await Task.Delay(TimeSpan.FromSeconds(4)); // if unexpected messages arrive, let them in to detect failure
_log.Info("Waiting for consumer.CloseAsync");
consumer.Dispose();
consumerSubscription.Dispose();
if (postCount + postCount2 != received.Count)
{
var receivedStr = received.Select(m => Encoding.UTF8.GetString(m.Value)).ToArray();
var diff = sent.Except(received.Select(m => Encoding.UTF8.GetString(m.Value))).OrderBy(s => s);
_log.Info("Not received {0}: \n {1}", diff.Count(), string.Join("\n ", diff));
var diff2 = sent.Except(confirmedSent1).OrderBy(s => s);
_log.Info("Not confirmed {0}: \n {1}", diff2.Count(), string.Join("\n ", diff2));
var diff3 = received.Select(m => Encoding.UTF8.GetString(m.Value)).Except(sent).OrderBy(s => s);
_log.Info("Received extra: {0}: \n {1}", diff3.Count(), string.Join("\n ", diff3));
var diff4 = confirmedSent1.Except(sent).OrderBy(s => s);
_log.Info("Confirmed extra {0}: \n {1}", diff4.Count(), string.Join("\n ", diff4));
var dups = receivedStr.GroupBy(s => s).Where(g => g.Count() > 1).Select(g => string.Format("{0}: {1}", g.Count(), g.Key));
_log.Info("Receved dups: \n {0}", string.Join("\n ", dups));
_log.Debug("Received: \n{0}", string.Join("\n ", received.Select(m => Encoding.UTF8.GetString(m.Value))));
}
Assert.AreEqual(postCount + postCount2, received.Count, "Received.Count");
_log.Info("Done");
kafka4net.Tracing.EtwTrace.Marker("/LeaderDownProducerAndConsumerRecovery");
}
public async void ListenerOnNonExistentTopicWaitsForTopicCreation()
{
kafka4net.Tracing.EtwTrace.Marker("ListenerOnNonExistentTopicWaitsForTopicCreation");
const int numMessages = 400;
var topic = "topic." + _rnd.Next();
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, new StartPositionTopicStart()));
var cancelSubject = new Subject<bool>();
var receivedValuesTask = consumer.OnMessageArrived
.Select(msg=>BitConverter.ToInt32(msg.Value, 0))
.Do(val=>_log.Info("Received: {0}", val))
.Take(numMessages)
.TakeUntil(cancelSubject)
.ToList().ToTask();
//receivedValuesTask.Start();
await consumer.IsConnected;
// wait a couple seconds for things to "stabilize"
await Task.Delay(TimeSpan.FromSeconds(4));
// now produce 400 messages
var producer = new Producer(_seed2Addresses, new ProducerConfiguration(topic));
await producer.ConnectAsync();
Enumerable.Range(1, numMessages).
Select(i => new Message { Value = BitConverter.GetBytes(i) }).
ForEach(producer.Send);
await Task.Delay(TimeSpan.FromSeconds(2));
await producer.CloseAsync(TimeSpan.FromSeconds(5));
// wait another little while, and stop the producer.
await Task.Delay(TimeSpan.FromSeconds(2));
cancelSubject.OnNext(true);
var receivedValues = await receivedValuesTask;
Assert.AreEqual(numMessages,receivedValues.Count);
kafka4net.Tracing.EtwTrace.Marker("/ListenerOnNonExistentTopicWaitsForTopicCreation");
}
public async void OneInvalidDnsShouldNotThrow()
{
var seed = "no.such.name.123.org,"+_seed2Addresses;
var consumer = new Consumer(new ConsumerConfiguration(seed, "some.topic", new StartPositionTopicEnd()));
consumer.OnMessageArrived.Subscribe(_ => { });
await consumer.IsConnected;
}
public async void SimulateLongBufferedMessageHandling()
{
var count = 2000;
var topic = "topic11." + _rnd.Next();
VagrantBrokerUtil.CreateTopic(topic, 1, 1);
var producer = new Producer(_seed2Addresses, new ProducerConfiguration(topic, batchFlushSize: 2));
await producer.ConnectAsync();
var src = Observable.Interval(TimeSpan.FromMilliseconds(10)).Take(count).
Select(i => new Message { Value = BitConverter.GetBytes((int)i) }).Publish().RefCount();
src.Subscribe(producer.Send);
await src;
await Task.Delay(200);
await producer.CloseAsync(TimeSpan.FromSeconds(5));
_log.Debug("Start consumer");
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, new StartPositionTopicStart()));
var stream = consumer.OnMessageArrived.
Buffer(TimeSpan.FromSeconds(1), count / 4).
Where(_ => _.Count > 0).
Select(i =>
{
_log.Debug("Handling batch {0} ...", i.Count);
Thread.Sleep(65*1000);
_log.Debug("Complete batch");
return i.Count;
}).
Scan(0, (i, i1) =>
{
_log.Debug("Scnning {0} {1}", i, i1);
return i + i1;
}).
Do(i => _log.Debug("Do {0}", i)).
Where(i => i == count).FirstAsync().ToTask();
await consumer.IsConnected;
_log.Debug("Awaiting for consumer");
var count2 = await stream;
consumer.Dispose();
Assert.AreEqual(count, count2);
_log.Info("Complete");
}
public async void KeyedMessagesPreserveOrder()
{
kafka4net.Tracing.EtwTrace.Marker("KeyedMessagesPreserveOrder");
// create a topic with 3 partitions
var topicName = "part33." + _rnd.Next();
VagrantBrokerUtil.CreateTopic(topicName, 3, 3);
// create listener in a separate connection/broker
var receivedMsgs = new List<ReceivedMessage>();
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topicName, new StartPositionTopicEnd()));
var consumerSubscription = consumer.OnMessageArrived.Synchronize().Subscribe(msg =>
{
lock (receivedMsgs)
{
receivedMsgs.Add(msg);
}
});
await consumer.IsConnected;
// sender is configured with 50ms batch period
var producer = new Producer(_seed2Addresses, new ProducerConfiguration(topicName, TimeSpan.FromMilliseconds(50)));
await producer.ConnectAsync();
//
// generate messages with 100ms interval in 10 threads
//
var sentMsgs = new List<Message>();
_log.Info("Start sending");
var senders = Enumerable.Range(1, 1).
Select(thread => Observable.
Interval(TimeSpan.FromMilliseconds(10)).
Synchronize(). // protect adding to sentMsgs
Select(i =>
{
var str = "msg " + i + " thread " + thread + " " + Guid.NewGuid();
var bin = Encoding.UTF8.GetBytes(str);
var msg = new Message
{
Key = BitConverter.GetBytes((int)(i + thread) % 10),
Value = bin
};
return Tuple.Create(msg, i, str);
}).
Subscribe(msg =>
{
lock (sentMsgs)
{
producer.Send(msg.Item1);
sentMsgs.Add(msg.Item1);
Assert.AreEqual(msg.Item2, sentMsgs.Count-1);
}
})
).
ToArray();
// wait for around 10K messages (10K/(10*10) = 100sec) and close producer
_log.Info("Waiting for producer to produce enough...");
await Task.Delay(100*1000);
_log.Info("Closing senders intervals");
senders.ForEach(s => s.Dispose());
_log.Info("Closing producer");
await producer.CloseAsync(TimeSpan.FromSeconds(5));
_log.Info("Waiting for additional 10sec");
await Task.Delay(10*1000);
_log.Info("Disposing consumer");
consumerSubscription.Dispose();
_log.Info("Closing consumer");
consumer.Dispose();
_log.Info("Done with networking");
// compare sent and received messages
// TODO: for some reason preformance is not what I'd expect it to be and only 6K is generated.
Assert.GreaterOrEqual(sentMsgs.Count, 4000, "Expected around 10K messages to be sent");
if (sentMsgs.Count != receivedMsgs.Count)
{
var sentStr = sentMsgs.Select(m => Encoding.UTF8.GetString(m.Value)).ToArray();
var receivedStr = receivedMsgs.Select(m => Encoding.UTF8.GetString(m.Value)).ToArray();
sentStr.Except(receivedStr).
ForEach(m => _log.Error("Not received: '{0}'", m));
receivedStr.Except(sentStr).
ForEach(m => _log.Error("Not sent but received: '{0}'", m));
}
Assert.AreEqual(sentMsgs.Count, receivedMsgs.Count, "Sent and received messages count differs");
//
// group messages by key and compare lists in each key to be the same (order should be preserved within key)
//
var keysSent = sentMsgs.GroupBy(m => BitConverter.ToInt32(m.Key, 0), m => Encoding.UTF8.GetString(m.Value), (i, mm) => new { Key = i, Msgs = mm.ToArray() }).ToArray();
var keysReceived = receivedMsgs.GroupBy(m => BitConverter.ToInt32(m.Key, 0), m => Encoding.UTF8.GetString(m.Value), (i, mm) => new { Key = i, Msgs = mm.ToArray() }).ToArray();
Assert.AreEqual(10, keysSent.Count(), "Expected 10 unique keys 0-9");
Assert.AreEqual(keysSent.Count(), keysReceived.Count(), "Keys count does not match");
// compare order within each key
var notInOrder = keysSent
.OrderBy(k => k.Key)
.Zip(keysReceived.OrderBy(k => k.Key), (s, r) => new { s, r, ok = s.Msgs.SequenceEqual(r.Msgs) }).Where(_ => !_.ok).ToArray();
if (notInOrder.Any())
{
_log.Error("{0} keys are out of order", notInOrder.Count());
notInOrder.ForEach(_ => _log.Error("Failed order in:\n{0}",
string.Join(" \n", DumpOutOfOrder(_.s.Msgs, _.r.Msgs))));
}
Assert.IsTrue(!notInOrder.Any(), "Detected out of order messages");
kafka4net.Tracing.EtwTrace.Marker("/KeyedMessagesPreserveOrder");
}
public async void StopAtExplicitOffsetOnEmptyTopic()
{
kafka4net.Tracing.EtwTrace.Marker("StopAtExplicitOffsetOnEmptyTopic");
// create new topic with 3 partitions
var topic = "part33." + _rnd.Next();
VagrantBrokerUtil.CreateTopic(topic, 3, 3);
var offsetFetchCluster = new Cluster(_seed2Addresses);
await offsetFetchCluster.ConnectAsync();
await Task.Delay(TimeSpan.FromSeconds(1));
var offsets = (await offsetFetchCluster.FetchPartitionOffsetsAsync(topic, ConsumerLocation.TopicEnd));
_log.Info("Sum of offsets {0}. Raw: {1}", offsets.Partitions.Sum(p => offsets.NextOffset(p)), offsets);
var startStopProvider = new StartAndStopAtExplicitOffsets(offsets, offsets);
_log.Info("Attempting to consume {0} messages and stop at {1}", 0, offsets);
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, startStopProvider, stopPosition: startStopProvider));
var startTime = DateTime.Now;
var timeout = startTime.AddSeconds(30);
var messages = await consumer.OnMessageArrived.TakeUntil(timeout).ToList();
_log.Info("Finished");
Assert.IsTrue(DateTime.Now < timeout);
Assert.AreEqual(0, messages.Count);
consumer.Dispose();
kafka4net.Tracing.EtwTrace.Marker("/StopAtExplicitOffsetOnEmptyTopic");
}
public async void SlowConsumer()
{
//
// 1. Create a topic with 100K messages.
// 2. Create slow consumer and start consuming at rate 1msg/sec
// 3. ???
//
var topic = "test32." + _rnd.Next();
await FillOutQueue(topic, (int)100e3);
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic,
new StartPositionTopicStart(),
maxBytesPerFetch: 1024,
lowWatermark:20, highWatermark: 200, useFlowControl: true));
var readWaiter = new SemaphoreSlim(0, 1);
var share = consumer.OnMessageArrived.Publish().RefCount();
var sub1 = share.Subscribe(msg =>
{
var str = Encoding.UTF8.GetString(msg.Value);
_log.Debug("Received msg '{0}'", str);
Thread.Sleep(100);
consumer.Ack();
}, e => _log.Error("Consumer error", e), () => readWaiter.Release());
await consumer.IsConnected;
// 2nd subscriber to test that Consumer does not count 2x of message delivery rate
var sub2 = share.Subscribe(_ => { }, e => { throw e; });
_log.Debug("Waiting for reader");
await readWaiter.WaitAsync();
_log.Debug("Reader complete");
}
public async void InvalidOffsetShouldLogErrorAndStopFetching()
{
var count = 100;
var topic = "test11."+_rnd.Next();
await FillOutQueue(topic, count);
var badPartitionMap = new Dictionary<int, long>{{0, -1}};
var offsets = new TopicPartitionOffsets(topic, badPartitionMap);
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, offsets));
await consumer.OnMessageArrived.Take(count);
await consumer.IsConnected;
_log.Info("Done");
}
public async void TwoConsumerSubscribersOneBroker()
{
kafka4net.Tracing.EtwTrace.Marker("TwoConsumerSubscribersOneBroker");
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, "part33", new StartPositionTopicEnd()));
var msgs = consumer.OnMessageArrived.Publish().RefCount();
var t1 = msgs.TakeUntil(DateTimeOffset.Now.AddSeconds(5)).LastOrDefaultAsync().ToTask();
var t2 = msgs.TakeUntil(DateTimeOffset.Now.AddSeconds(6)).LastOrDefaultAsync().ToTask();
//await consumer.IsConnected;
await Task.WhenAll(new[] { t1, t2 });
consumer.Dispose();
kafka4net.Tracing.EtwTrace.Marker("/TwoConsumerSubscribersOneBroker");
}
public async void SchedulerThreadIsIsolatedFromUserCode()
{
kafka4net.Tracing.EtwTrace.Marker("SchedulerThreadIsIsolatedFromUserCode");
const string threadName = "kafka-scheduler";
_log.Info("Test Runner is using thread {0}", Thread.CurrentThread.Name);
var topic = "topic." + _rnd.Next();
VagrantBrokerUtil.CreateTopic(topic,6,3);
var cluster = new Cluster(_seed2Addresses);
await cluster.ConnectAsync();
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
await cluster.FetchPartitionOffsetsAsync(topic, ConsumerLocation.TopicStart);
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
var topics = await cluster.GetAllTopicsAsync();
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
// now create a producer
var producer = new Producer(cluster, new ProducerConfiguration(topic));
await producer.ConnectAsync();
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
// create a producer that also creates a cluster
var producerWithCluster = new Producer(_seed2Addresses, new ProducerConfiguration(topic));
await producerWithCluster.ConnectAsync();
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
// TODO: Subscribe and check thread on notification observables!
// run them both for a little while (~5 seconds)
var msgs = await Observable.Interval(TimeSpan.FromMilliseconds(100))
.Do(l =>
{
producer.Send(new Message { Value = BitConverter.GetBytes(l) });
producerWithCluster.Send(new Message { Value = BitConverter.GetBytes(l) });
_log.Debug("After Producer Send using thread {0}", Thread.CurrentThread.Name);
}).Take(50).ToArray();
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
// now consumer(s)
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, new StartPositionTopicStart()));
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
var msgsRcv = new List<long>();
var messageSubscription = consumer.OnMessageArrived
.Do(msg => Assert.AreEqual(threadName, Thread.CurrentThread.Name), exception => Assert.AreEqual(threadName, Thread.CurrentThread.Name), () => Assert.AreEqual(threadName, Thread.CurrentThread.Name))
.Take(50)
.TakeUntil(DateTime.Now.AddSeconds(500))
.ObserveOn(System.Reactive.Concurrency.DefaultScheduler.Instance)
.Do(msg => Assert.AreNotEqual(threadName, Thread.CurrentThread.Name), exception => Assert.AreNotEqual(threadName, Thread.CurrentThread.Name), () => Assert.AreNotEqual(threadName, Thread.CurrentThread.Name))
.Subscribe(
msg=>
{
msgsRcv.Add(BitConverter.ToInt64(msg.Value,0));
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
_log.Debug("In Consumer Subscribe OnNext using thread {0}", Thread.CurrentThread.Name);
}, exception =>
{
_log.Debug("In Consumer Subscribe OnError using thread {0} Error: {1}", Thread.CurrentThread.Name, exception.Message);
throw exception;
}, () =>
{
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
_log.Debug("In Consumer Subscribe OnComplete using thread {0}", Thread.CurrentThread.Name);
});
await consumer.IsConnected;
_log.Info("Waitng for consumer to read");
await Task.Delay(TimeSpan.FromSeconds(6));
_log.Debug("After Consumer Subscribe using thread {0}", Thread.CurrentThread.Name);
consumer.Dispose();
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
Assert.AreEqual(msgs.Length, msgsRcv.Count);
messageSubscription.Dispose();
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
// now close down
await producer.CloseAsync(TimeSpan.FromSeconds(5));
_log.Debug("After Consumer Close using thread {0}", Thread.CurrentThread.Name);
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
await producerWithCluster.CloseAsync(TimeSpan.FromSeconds(5));
_log.Debug("After Producer Subscribe using thread {0}", Thread.CurrentThread.Name);
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
await cluster.CloseAsync(TimeSpan.FromSeconds(5));
_log.Debug("After Cluster Close using thread {0}", Thread.CurrentThread.Name);
Assert.AreNotEqual(threadName, Thread.CurrentThread.Name);
kafka4net.Tracing.EtwTrace.Marker("/SchedulerThreadIsIsolatedFromUserCode");
}
public async void SaveOffsetsAndResumeConsuming()
{
kafka4net.Tracing.EtwTrace.Marker("SaveOffsetsAndResumeConsuming");
var sentEvents = new Subject<Message>();
var topic = "part12." + _rnd.Next();
VagrantBrokerUtil.CreateTopic(topic, 5, 2);
var producer = new Producer(_seed2Addresses, new ProducerConfiguration(topic));
producer.OnSuccess += e => e.ForEach(sentEvents.OnNext);
await producer.ConnectAsync();
// send 100 messages
Enumerable.Range(1, 100).
Select(i => new Message { Value = BitConverter.GetBytes(i) }).
ForEach(producer.Send);
_log.Info("Waiting for 100 sent messages");
sentEvents.Subscribe(msg => _log.Debug("Sent {0}", BitConverter.ToInt32(msg.Value, 0)));
await sentEvents.Take(100).ToTask();
var offsets1 = await producer.Cluster.FetchPartitionOffsetsAsync(topic, ConsumerLocation.TopicStart);
_log.Info("Closing producer");
await producer.CloseAsync(TimeSpan.FromSeconds(5));
// now consume the "first" 50. Stop, save offsets, and restart.
var consumer1 = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, offsets1));
var receivedEvents = new List<int>(100);
_log.Info("Consuming first half of messages.");
await consumer1.OnMessageArrived
.Do(msg =>
{
var value = BitConverter.ToInt32(msg.Value, 0);
_log.Info("Consumer1 Received value {0} from partition {1} at offset {2}", value, msg.Partition, msg.Offset);
receivedEvents.Add(value);
offsets1.UpdateOffset(msg.Partition, msg.Offset);
})
.Take(50);
//await consumer1.IsConnected;
_log.Info("Closing first consumer");
consumer1.Dispose();
// now serialize the offsets.
var offsetBytes = offsets1.WriteOffsets();
// load a new set of offsets, and a new consumer
var offsets2 = new TopicPartitionOffsets(offsetBytes);
var consumer2 = new Consumer(new ConsumerConfiguration(_seed2Addresses, offsets2.Topic, offsets2));
await consumer2.OnMessageArrived
.Do(msg =>
{
var value = BitConverter.ToInt32(msg.Value, 0);
_log.Info("Consumer2 Received value {0} from partition {1} at offset {2}", value, msg.Partition, msg.Offset);
receivedEvents.Add(value);
offsets2.UpdateOffset(msg.Partition, msg.Offset);
})
.Take(50);
//await consumer2.IsConnected;
_log.Info("Closing second consumer");
consumer2.Dispose();
Assert.AreEqual(100, receivedEvents.Distinct().Count());
Assert.AreEqual(100, receivedEvents.Count);
kafka4net.Tracing.EtwTrace.Marker("/SaveOffsetsAndResumeConsuming");
}
public async void TopicIsAutocreatedByProducer()
{
kafka4net.Tracing.EtwTrace.Marker("TopicIsAutocreatedByProducer");
var topic ="autocreate.test." + _rnd.Next();
const int producedCount = 10;
var lala = Encoding.UTF8.GetBytes("la-la-la");
// TODO: set wait to 5sec
//
// Produce
// In order to make sure that topic was created by producer, send and wait for producer
// completion before performing validation read.
//
var producer = new Producer(_seed2Addresses, new ProducerConfiguration(topic));
await producer.ConnectAsync();
_log.Debug("Producing...");
await Observable.Interval(TimeSpan.FromSeconds(1)).
Take(producedCount).
Do(_ => producer.Send(new Message { Value = lala })).
ToTask();
await producer.CloseAsync(TimeSpan.FromSeconds(10));
//
// Validate by reading published messages
//
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, new StartPositionTopicStart(), maxWaitTimeMs: 1000, minBytesPerFetch: 1));
var msgs = consumer.OnMessageArrived.Publish().RefCount();
var receivedTxt = new List<string>();
var consumerSubscription = msgs.
Select(m => Encoding.UTF8.GetString(m.Value)).
Synchronize(). // protect receivedTxt
Do(m => _log.Info("Received {0}", m)).
Do(receivedTxt.Add).
Subscribe();
await consumer.IsConnected;
_log.Debug("Waiting for consumer");
await msgs.Take(producedCount).TakeUntil(DateTimeOffset.Now.AddSeconds(5)).LastOrDefaultAsync().ToTask();
Assert.AreEqual(producedCount, receivedTxt.Count, "Did not received all messages");
Assert.IsTrue(receivedTxt.All(m => m == "la-la-la"), "Unexpected message content");
consumerSubscription.Dispose();
consumer.Dispose();
kafka4net.Tracing.EtwTrace.Marker("/TopicIsAutocreatedByProducer");
}
public async void ReadFromHead()
{
kafka4net.Tracing.EtwTrace.Marker("ReadFromHead");
const int count = 100;
var topic = "part32." + _rnd.Next();
VagrantBrokerUtil.CreateTopic(topic,3,2);
// fill it out with 100 messages
var producer = new Producer(_seed2Addresses, new ProducerConfiguration(topic));
await producer.ConnectAsync();
_log.Info("Sending data");
Enumerable.Range(1, count).
Select(i => new Message { Value = BitConverter.GetBytes(i) }).
ForEach(producer.Send);
_log.Debug("Closing producer");
await producer.CloseAsync(TimeSpan.FromSeconds(5));
// read starting from the head
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, new StartPositionTopicStart()));
var count2 = await consumer.OnMessageArrived.TakeUntil(DateTimeOffset.Now.AddSeconds(5))
//.Do(val=>_log.Info("received value {0}", BitConverter.ToInt32(val.Value,0)))
.Count().ToTask();
//await consumer.IsConnected;
Assert.AreEqual(count, count2);
kafka4net.Tracing.EtwTrace.Marker("/ReadFromHead");
}
public async void CleanShutdownTest()
{
kafka4net.Tracing.EtwTrace.Marker("CleanShutdownTest");
const string topic = "shutdown.test";
// set producer long batching period, 20 sec
var producer = new Producer(_seed2Addresses, new ProducerConfiguration(topic, TimeSpan.FromSeconds(20), int.MaxValue));
_log.Debug("Connecting");
await producer.ConnectAsync();
// start listener at the end of queue and accumulate received messages
var received = new HashSet<string>();
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, new StartPositionTopicEnd(), maxWaitTimeMs: 30 * 1000));
_log.Info("Subscribing to consumer");
var consumerSubscription = consumer.OnMessageArrived
.Select(msg => Encoding.UTF8.GetString(msg.Value))
.Subscribe(m => received.Add(m));
_log.Info("Connecting consumer");
await consumer.IsConnected;
_log.Info("Subscribed to consumer");
_log.Info("Starting sender");
// send data, 5 msg/sec, for 5 seconds
var sent = new HashSet<string>();
var sender = Observable.Interval(TimeSpan.FromSeconds(1.0 / 5)).
Select(i => string.Format("msg {0} {1}", i, Guid.NewGuid())).
Synchronize().
Do(m => sent.Add(m)).
Select(msg => new Message
{
Value = Encoding.UTF8.GetBytes(msg)
}).
TakeUntil(DateTimeOffset.Now.AddSeconds(5)).
Publish().RefCount();
sender.Subscribe(producer.Send);
_log.Debug("Waiting for sender");
await sender;
_log.Debug("Waiting for producer complete");
await producer.CloseAsync(TimeSpan.FromSeconds(4));
// how to make sure nothing is sent after shutdown? listen to logger? have connection events?
// wait for 5sec for receiver to get all the messages
_log.Info("Waiting for consumer to fetch");
await Task.Delay(5000);
_log.Info("Disposing consumer subscription");
consumerSubscription.Dispose();
_log.Info("Closing consumer");
consumer.Dispose();
_log.Info("Closed consumer");
// assert we received all the messages
Assert.AreEqual(sent.Count, received.Count, string.Format("Sent and Receved size differs. Sent: {0} Recevied: {1}", sent.Count, received.Count));
// compare sets and not lists, because of 2 partitions, send order and receive orser are not the same
Assert.True(received.SetEquals(sent), "Sent and Received set differs");
kafka4net.Tracing.EtwTrace.Marker("/CleanShutdownTest");
}
public async void SpeedTest()
{
int _pageViewBatchSize = 10 * 1000;
TimeSpan _batchTime = TimeSpan.FromSeconds(10);
var conf = new ConsumerConfiguration("kafkadev-01.lv.ntent.com",
"vsw.avrodto.addelivery.activitylogging.pageview",
new StartPositionTopicStart(),
useFlowControl: true,
highWatermark: _pageViewBatchSize * 5,
lowWatermark: _pageViewBatchSize * 2
);
var consumer = new Consumer(conf);
var count = 0L;
var lastCount = count;
var msg = consumer.OnMessageArrived.Publish().RefCount();
msg.
Buffer(_batchTime, _pageViewBatchSize).
Subscribe(_ =>
{
Interlocked.Add(ref count, _.Count);
consumer.Ack(_.Count);
//consumer.Ack();
});
var interval = TimeSpan.FromSeconds(5);
var lastTime = DateTime.Now;
Observable.Interval(interval).Subscribe(_ => {
var c = count;
var now = DateTime.Now;
var time = now - lastTime;
var speed = (c - lastCount)/time.TotalSeconds;
lastCount = c;
lastTime = now;
Console.WriteLine("{0}msg/sec #{1}", speed, c);
});
await consumer.IsConnected;
await msg.Timeout(TimeSpan.FromSeconds(30));
Console.WriteLine("Complete {0}", count);
}
public async void ConsumerFollowsRebalancingPartitions()
{
kafka4net.Tracing.EtwTrace.Marker("ConsumerFollowsRebalancingPartitions");
// create a topic
var topic = "topic33." + _rnd.Next();
VagrantBrokerUtil.CreateTopic(topic,11,3);
// Stop two brokers to let leadership shift to broker1.
VagrantBrokerUtil.StopBroker("broker2");
VagrantBrokerUtil.StopBroker("broker3");
await Task.Delay(TimeSpan.FromSeconds(5));
// now start back up
VagrantBrokerUtil.StartBroker("broker2");
VagrantBrokerUtil.StartBroker("broker3");
// wait a little for everything to start
await Task.Delay(TimeSpan.FromSeconds(5));
// we should have all of them with leader 1
var cluster = new Cluster(_seed2Addresses);
await cluster.ConnectAsync();
var partitionMeta = await cluster.GetOrFetchMetaForTopicAsync(topic);
// make sure they're all on a single leader
Assert.AreEqual(1, partitionMeta.GroupBy(p=>p.Leader).Count());
// now publish messages
const int count = 25000;
var producer = new Producer(cluster, new ProducerConfiguration(topic));
_log.Debug("Connecting");
await producer.ConnectAsync();
_log.Debug("Filling out {0} with {1} messages", topic, count);
var sentList = await Enumerable.Range(0, count)
.Select(i => new Message { Value = BitConverter.GetBytes(i) })
.ToObservable()
.Do(producer.Send)
.Select(msg => BitConverter.ToInt32(msg.Value, 0))
.ToList();
await Task.Delay(TimeSpan.FromSeconds(1));
_log.Info("Done sending messages. Closing producer.");
await producer.CloseAsync(TimeSpan.FromSeconds(5));
_log.Info("Producer closed, starting consumer subscription.");
await Task.Delay(TimeSpan.FromSeconds(1));
var heads = await cluster.FetchPartitionOffsetsAsync(topic, ConsumerLocation.TopicStart);
var tails = await cluster.FetchPartitionOffsetsAsync(topic, ConsumerLocation.TopicEnd);
var messagesInTopic = (int)tails.MessagesSince(heads);
_log.Info("Topic offsets indicate producer sent {0} messages.", messagesInTopic);
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, new StartPositionTopicStart(), maxBytesPerFetch: 4 * 8));
var current = 0;
var received = new ReplaySubject<ReceivedMessage>();
Task rebalanceTask = null;
var consumerSubscription = consumer.OnMessageArrived.
Subscribe(async msg =>
{
current++;
if (current == 18)
{
rebalanceTask = Task.Factory.StartNew(VagrantBrokerUtil.RebalanceLeadership, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default);
}
received.OnNext(msg);
//_log.Info("Got: {0}", BitConverter.ToInt32(msg.Value, 0));
});
await consumer.IsConnected;
_log.Info("Waiting for receiver complete");
var receivedList = await received.Select(msg => BitConverter.ToInt32(msg.Value, 0)).
Take(messagesInTopic).
TakeUntil(DateTime.Now.AddMinutes(3)).
ToList().
ToTask();
if (rebalanceTask != null)
{
_log.Info("Waiting for rebalance complete");
await rebalanceTask;//.TimeoutAfter(TimeSpan.FromSeconds(10));
_log.Info("Rebalance complete");
}
_log.Info("Receiver complete. Disposing Subscription");
consumerSubscription.Dispose();
_log.Info("Consumer subscription disposed. Closing consumer.");
consumer.Dispose();
_log.Info("Consumer closed.");
tails = await cluster.FetchPartitionOffsetsAsync(topic, ConsumerLocation.TopicEnd);
await cluster.CloseAsync(TimeSpan.FromSeconds(5));
_log.Info("Sum of offsets: {0}", tails.MessagesSince(heads));
_log.Info("Offsets: [{0}]", string.Join(",", tails.Partitions.Select(p => string.Format("{0}:{1}", p, tails.NextOffset(p)))));
if (messagesInTopic != receivedList.Count)
{
// log some debug info.
_log.Error("Did not receive all messages. Messages sent but NOT received: {0}", string.Join(",", sentList.Except(receivedList).OrderBy(i => i)));
}
Assert.AreEqual(messagesInTopic, receivedList.Count);
kafka4net.Tracing.EtwTrace.Marker("/ConsumerFollowsRebalancingPartitions");
}
public async void InvalidDnsShouldThrowException()
{
var consumer = new Consumer(new ConsumerConfiguration("no.such.name.123.org", "some.topic", new StartPositionTopicEnd()));
consumer.OnMessageArrived.Subscribe(_ => { });
await consumer.IsConnected;
}
public async void ExplicitOffset()
{
kafka4net.Tracing.EtwTrace.Marker("ExplicitOffset");
// create new topic with 3 partitions
var topic = "part33." + _rnd.Next();
VagrantBrokerUtil.CreateTopic(topic,3,3);
// fill it out with 10K messages
const int count = 10*1000;
var producer = new Producer(_seed2Addresses, new ProducerConfiguration(topic));
await producer.ConnectAsync();
var sentMessagesObservable = Observable.FromEvent<Message[]>(evtHandler => producer.OnSuccess += evtHandler, evtHandler => { })
.SelectMany(msgs=>msgs)
.Take(count)
.TakeUntil(DateTime.Now.AddSeconds(10))
.ToList();
_log.Info("Sending data");
Enumerable.Range(1, count).
Select(i => new Message { Value = BitConverter.GetBytes(i) }).
ForEach(producer.Send);
var sentMsgs = await sentMessagesObservable;
_log.Info("Producer sent {0} messages.", sentMsgs.Count);
_log.Debug("Closing producer");
await producer.CloseAsync(TimeSpan.FromSeconds(5));
var offsetFetchCluster = new Cluster(_seed2Addresses);
await offsetFetchCluster.ConnectAsync();
// consume tail-300 for each partition
await Task.Delay(TimeSpan.FromSeconds(1));
var offsets = new TopicPartitionOffsets(
topic, (await offsetFetchCluster.FetchPartitionOffsetsAsync(topic, ConsumerLocation.TopicEnd))
.GetPartitionsOffset.Select(kv=>new KeyValuePair<int,long>(kv.Key,kv.Value-300)));
_log.Info("Sum of offsets {0}. Raw: {1}",offsets.Partitions.Sum(p=>offsets.NextOffset(p)), offsets);
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, offsets));
var messages = consumer.OnMessageArrived.
GroupBy(m => m.Partition).Replay();
messages.Connect();
await consumer.IsConnected;
var consumerSubscription = messages.Subscribe(p => p.Take(10).Subscribe(
m => _log.Debug("Got message {0}/{1}", m.Partition, BitConverter.ToInt32(m.Value, 0)),
e => _log.Error("Error", e),
() => _log.Debug("Complete part {0}", p.Key)
));
// wait for 3 partitions to arrrive and every partition to read at least 100 messages
await messages.Select(g => g.Take(100)).Take(3).ToTask();
consumerSubscription.Dispose();
consumer.Dispose();
kafka4net.Tracing.EtwTrace.Marker("/ExplicitOffset");
}
public async void StartAndStopAtExplicitOffset()
{
kafka4net.Tracing.EtwTrace.Marker("StartAndStopAtExplicitOffset");
// create new topic with 3 partitions
var topic = "part33." + _rnd.Next();
VagrantBrokerUtil.CreateTopic(topic, 3, 3);
// fill it out with 10K messages
const int count = 10 * 1000;
var producer = new Producer(_seed2Addresses, new ProducerConfiguration(topic));
await producer.ConnectAsync();
var sentMessagesObservable = Observable.FromEvent<Message[]>(evtHandler => producer.OnSuccess += evtHandler, evtHandler => { })
.SelectMany(msgs => msgs)
.Take(count)
.TakeUntil(DateTime.Now.AddSeconds(10))
.ToList();
_log.Info("Sending data");
Enumerable.Range(1, count).
Select(i => new Message { Value = BitConverter.GetBytes(i) }).
ForEach(producer.Send);
var sentMsgs = await sentMessagesObservable;
_log.Info("Producer sent {0} messages.", sentMsgs.Count);
_log.Debug("Closing producer");
await producer.CloseAsync(TimeSpan.FromSeconds(5));
var offsetFetchCluster = new Cluster(_seed2Addresses);
await offsetFetchCluster.ConnectAsync();
await Task.Delay(TimeSpan.FromSeconds(1));
var offsets = (await offsetFetchCluster.FetchPartitionOffsetsAsync(topic, ConsumerLocation.TopicEnd));
_log.Info("Sum of offsets {0}. Raw: {1}", offsets.Partitions.Sum(p => offsets.NextOffset(p)), offsets);
// consume first 300 for each partition
var offsetStarts = new TopicPartitionOffsets(topic, offsets.GetPartitionsOffset.ToDictionary(pair => pair.Key, pair => pair.Value > 300 ? 300 : pair.Value));
var offsetStops = new TopicPartitionOffsets(topic, offsets.GetPartitionsOffset.ToDictionary(pair => pair.Key, pair => pair.Value > 600 ? 600 : pair.Value));
var numMessages = offsetStops.MessagesSince(offsetStarts);
var startStopProvider = new StartAndStopAtExplicitOffsets(offsetStarts, offsetStops);
_log.Info("Attempting to consume {0} messages and stop at {1}", numMessages, offsetStops);
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic, startStopProvider, stopPosition: startStopProvider));
var messages = await consumer.OnMessageArrived.ToList();
consumer.Dispose();
Assert.AreEqual(numMessages, messages.Count);
kafka4net.Tracing.EtwTrace.Marker("/StartAndStopAtExplicitOffset");
}