public async void SpeedTest()
{
int _pageViewBatchSize = 10 * 1000;
TimeSpan _batchTime = TimeSpan.FromSeconds(10);
var conf = new ConsumerConfiguration("kafkadev-01.lv.ntent.com",
"vsw.avrodto.addelivery.activitylogging.pageview",
new StartPositionTopicStart(),
useFlowControl: true,
highWatermark: _pageViewBatchSize * 5,
lowWatermark: _pageViewBatchSize * 2
);
var consumer = new Consumer(conf);
var count = 0L;
var lastCount = count;
var msg = consumer.OnMessageArrived.Publish().RefCount();
msg.
Buffer(_batchTime, _pageViewBatchSize).
Subscribe(_ =>
{
Interlocked.Add(ref count, _.Count);
consumer.Ack(_.Count);
//consumer.Ack();
});
var interval = TimeSpan.FromSeconds(5);
var lastTime = DateTime.Now;
Observable.Interval(interval).Subscribe(_ => {
var c = count;
var now = DateTime.Now;
var time = now - lastTime;
var speed = (c - lastCount)/time.TotalSeconds;
lastCount = c;
lastTime = now;
Console.WriteLine("{0}msg/sec #{1}", speed, c);
});
await consumer.IsConnected;
await msg.Timeout(TimeSpan.FromSeconds(30));
Console.WriteLine("Complete {0}", count);
}
public async void SlowConsumer()
{
//
// 1. Create a topic with 100K messages.
// 2. Create slow consumer and start consuming at rate 1msg/sec
// 3. ???
//
var topic = "test32." + _rnd.Next();
await FillOutQueue(topic, (int)100e3);
var consumer = new Consumer(new ConsumerConfiguration(_seed2Addresses, topic,
new StartPositionTopicStart(),
maxBytesPerFetch: 1024,
lowWatermark:20, highWatermark: 200, useFlowControl: true));
var readWaiter = new SemaphoreSlim(0, 1);
var share = consumer.OnMessageArrived.Publish().RefCount();
var sub1 = share.Subscribe(msg =>
{
var str = Encoding.UTF8.GetString(msg.Value);
_log.Debug("Received msg '{0}'", str);
Thread.Sleep(100);
consumer.Ack();
}, e => _log.Error("Consumer error", e), () => readWaiter.Release());
await consumer.IsConnected;
// 2nd subscriber to test that Consumer does not count 2x of message delivery rate
var sub2 = share.Subscribe(_ => { }, e => { throw e; });
_log.Debug("Waiting for reader");
await readWaiter.WaitAsync();
_log.Debug("Reader complete");
}