private void ListenToConsole(Kafka.Basic.BatchedConsumer consumer)
{
_consoleThread = new Thread(() =>
{
Console.CancelKeyPress += (sender, eventArgs) =>
{
Console.WriteLine("Kill!");
_consoleThread.Abort();
};
while (true)
{
var input = Console.ReadKey(true);
//if (input.KeyChar == 'p')
//{
// stream.Pause();
// Console.WriteLine("Paused.");
//}
//if (input.KeyChar == 'r')
//{
// stream.Resume();
// Console.WriteLine("Resumed.");
//}
if (input.KeyChar == 'q')
{
Console.WriteLine("Shutting down...");
consumer.Shutdown();
break;
}
}
});
_consoleThread.Start();
}
public int Start(BatchedConsumerOptions opts)
{
var histogram = Metric.Histogram("batch.size", Unit.Items);
var timer = Metric.Timer("message.latency", Unit.Events);
Metric.Config.WithReporting(r => r.WithConsoleReport(TimeSpan.FromSeconds(5)));
using (var consumer = new Kafka.Basic.BatchedConsumer(opts.ZkConnect, opts.Group, opts.Topic, opts.Threads, opts.BatchTimeoutMs))
{
ListenToConsole(consumer);
consumer
.Start(m =>
{
var time = DateTime.UtcNow.Ticks;
var list = m.ToList();
list
.ForEach(message =>
{
var value = long.Parse(message.Value);
var diff = (time - value) / 10000;
timer.Record(diff, TimeUnit.Milliseconds);
});
histogram.Update(list.Count);
});
}
return(0);
}
public int Start(ChaosMonkeyOptions opts)
{
var histogram = Metric.Histogram("batch.size", Unit.Items);
var timer = Metric.Timer("message.latency", Unit.Events);
var random = new Random();
var exceptionModulus = random.Next(76, 100);
var waitModulus = random.Next(50, 75);
using (var consumer = new Kafka.Basic.BatchedConsumer(opts.ZkConnect, opts.Group, opts.Topic, opts.Threads, opts.BatchTimeoutMs, opts.MaxBatchSize))
{
var call = 0;
consumer
.Start(
m =>
{
var time = DateTime.UtcNow.Ticks;
call++;
var list = m.ToList();
WriteLog($"Received batch {call} of {list.Count} messages:");
list
.ForEach(message =>
{
var value = long.Parse(message.Value);
var diff = (time - value) / 10000;
timer.Record(diff, TimeUnit.Milliseconds);
WriteLog($"P:{message.Partition},O:{message.Offset},L:{diff}ms");
});
if (call % exceptionModulus == 0)
{
WriteLog($"Faking an exception to test shutdown behaviour... Expect to see the above {list.Count} messages come through again.");
throw new ChaosException();
}
if (call % waitModulus == 0)
{
WriteLog("Faking a long consume operation by sending the current thread to sleep for 5s.");
Thread.Sleep(5000);
}
histogram.Update(list.Count);
},
e =>
{
if (e is ChaosException)
{
WriteLog("Chaos achieved and handled.");
}
else
{
WriteLog("Got an error... ", e);
}
});
}
return(0);
}
