public async Task OffsetCommitShouldStoreAndReturnSuccess()
{
const int partitionId = 0;
var router = new BrokerRouter(_options);
await router.GetTopicMetadataAsync(TestConfig.TopicName(), CancellationToken.None);
var conn = router.GetBrokerRoute(TestConfig.TopicName(), partitionId);
// ensure the group exists
var group = new GroupCoordinatorRequest(TestConfig.ConsumerName());
var groupResponse = await conn.Connection.SendAsync(group, CancellationToken.None);
Assert.That(groupResponse, Is.Not.Null);
Assert.That(groupResponse.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
var commit = new OffsetCommitRequest(group.GroupId, new [] { new OffsetCommitRequest.Topic(TestConfig.TopicName(), partitionId, 10, null) });
var response = await conn.Connection.SendAsync(commit, CancellationToken.None);
var topic = response.Topics.FirstOrDefault();
Assert.That(topic, Is.Not.Null);
Assert.That(topic.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
router.Dispose();
}
public void OffsetCommitShouldStoreMetadata()
{
const int partitionId = 0;
const long offset = 101;
const string metadata = "metadata";
var router = new BrokerRouter(Options);
var conn = router.SelectBrokerRouteFromLocalCache(IntegrationConfig.IntegrationTopic, partitionId);
var commit = CreateOffsetCommitRequest(IntegrationConfig.IntegrationConsumer, partitionId, offset, metadata);
var commitResponse = conn.Connection.SendAsync(commit).Result.FirstOrDefault();
Assert.That(commitResponse, Is.Not.Null);
Assert.That(commitResponse.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
var fetch = CreateOffsetFetchRequest(IntegrationConfig.IntegrationConsumer, partitionId);
var fetchResponse = conn.Connection.SendAsync(fetch).Result.FirstOrDefault();
Assert.That(fetchResponse, Is.Not.Null);
Assert.That(fetchResponse.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
Assert.That(fetchResponse.Offset, Is.EqualTo(offset));
Assert.That(fetchResponse.MetaData, Is.EqualTo(metadata));
router.Dispose();
}
public async Task OffsetCommitShouldStoreOffsetValue()
{
const int partitionId = 0;
const long offset = 99;
var router = new BrokerRouter(Options);
await router.RefreshMissingTopicMetadata(IntegrationConfig.IntegrationTopic);
var conn = router.SelectBrokerRouteFromLocalCache(IntegrationConfig.IntegrationTopic, partitionId);
var commit = CreateOffsetCommitRequest(IntegrationConfig.IntegrationConsumer, partitionId, offset);
var commitResponse = (await conn.Connection.SendAsync(commit)).FirstOrDefault();
Assert.That(commitResponse, Is.Not.Null);
Assert.That(commitResponse.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
var fetch = CreateOffsetFetchRequest(IntegrationConfig.IntegrationConsumer, partitionId);
var fetchResponse = (await conn.Connection.SendAsync(fetch)).FirstOrDefault();
Assert.That(fetchResponse, Is.Not.Null);
Assert.That(fetchResponse.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
Assert.That(fetchResponse.Offset, Is.EqualTo(offset));
router.Dispose();
}
/// <summary>
/// Attempts to disconnect from data output stream.
/// </summary>
/// <remarks>
/// Derived classes should attempt disconnect from data output stream here. Any exceptions thrown
/// by this implementation will be reported to host via <see cref="AdapterBase.ProcessException"/> event.
/// </remarks>
protected override void AttemptDisconnection()
{
if ((object)m_producer != null)
{
m_producer.Dispose();
m_producer = null;
}
if ((object)m_router != null)
{
m_router.Dispose();
m_router = null;
}
}
public async Task OffsetFetchRequestOfNonExistingGroupShouldReturnNoError()
{
//From documentation: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+ProtocolTests#AGuideToTheKafkaProtocol-OffsetFetchRequest
//Note that if there is no offset associated with a topic-partition under that consumer group the broker does not set an error code
//(since it is not really an error), but returns empty metadata and sets the offset field to -1.
const int partitionId = 0;
var router = new BrokerRouter(Options);
var request = CreateOffsetFetchRequest(Guid.NewGuid().ToString(), partitionId);
await router.RefreshMissingTopicMetadata(IntegrationConfig.IntegrationTopic);
var conn = router.SelectBrokerRouteFromLocalCache(IntegrationConfig.IntegrationTopic, partitionId);
var response = (await conn.Connection.SendAsync(request)).FirstOrDefault();
Assert.That(response, Is.Not.Null);
Assert.That(response.Error, Is.EqualTo((int)ErrorResponseCode.NoError));
Assert.That(response.Offset, Is.EqualTo(-1));
router.Dispose();
}
/// <summary>
/// Attempts to disconnect from data output stream.
/// </summary>
/// <remarks>
/// Derived classes should attempt disconnect from data output stream here. Any exceptions thrown
/// by this implementation will be reported to host via <see cref="AdapterBase.ProcessException"/> event.
/// </remarks>
protected override void AttemptDisconnection()
{
if ((object)m_processingThreads != null)
{
foreach (Thread processingThread in m_processingThreads)
{
processingThread.Abort();
}
m_processingThreads = null;
}
if ((object)m_router != null)
{
m_router.Dispose();
m_router = null;
}
lock (m_consumers)
m_consumers.Clear();
}
public async Task OffsetFetchRequestOfNonExistingGroupShouldReturnNoError()
{
//From documentation: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+ProtocolTests#AGuideToTheKafkaProtocol-OffsetFetchRequest
//Note that if there is no offset associated with a topic-partition under that consumer group the broker does not set an error code
//(since it is not really an error), but returns empty metadata and sets the offset field to -1.
const int partitionId = 0;
var router = new BrokerRouter(_options);
var request = new OffsetFetchRequest(Guid.NewGuid().ToString(), new TopicPartition(TestConfig.TopicName(), partitionId));
await router.GetTopicMetadataAsync(TestConfig.TopicName(), CancellationToken.None);
var conn = router.GetBrokerRoute(TestConfig.TopicName(), partitionId);
var response = await conn.Connection.SendAsync(request, CancellationToken.None);
var topic = response.Topics.FirstOrDefault();
Assert.That(topic, Is.Not.Null);
Assert.That(topic.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
Assert.That(topic.Offset, Is.EqualTo(-1));
router.Dispose();
}
public async Task OffsetCommitShouldStoreOffsetValue()
{
const int partitionId = 0;
const long offset = 99;
var router = new BrokerRouter(_options);
await router.GetTopicMetadataAsync(TestConfig.TopicName(), CancellationToken.None);
var conn = router.GetBrokerRoute(TestConfig.TopicName(), partitionId);
// ensure the group exists
var group = new GroupCoordinatorRequest(TestConfig.ConsumerName());
var groupResponse = await conn.Connection.SendAsync(group, CancellationToken.None);
Assert.That(groupResponse, Is.Not.Null);
Assert.That(groupResponse.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
var commit = new OffsetCommitRequest(group.GroupId, new [] { new OffsetCommitRequest.Topic(TestConfig.TopicName(), partitionId, offset, null) });
var commitResponse = await conn.Connection.SendAsync(commit, CancellationToken.None);
var commitTopic = commitResponse.Topics.SingleOrDefault();
Assert.That(commitTopic, Is.Not.Null);
Assert.That(commitTopic.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
var fetch = new OffsetFetchRequest(TestConfig.ConsumerName(), new TopicPartition(TestConfig.TopicName(), partitionId));
var fetchResponse = await conn.Connection.SendAsync(fetch, CancellationToken.None);
var fetchTopic = fetchResponse.Topics.SingleOrDefault();
Assert.That(fetchTopic, Is.Not.Null);
Assert.That(fetchTopic.ErrorCode, Is.EqualTo(ErrorResponseCode.None));
Assert.That(fetchTopic.Offset, Is.EqualTo(offset));
router.Dispose();
}
public void Dispose()
{
_router.Dispose();
}
public async Task ConsumerShouldConsumeInSameOrderAsAsyncProduced_dataLoad(int numberOfMessage, int timeoutInMs)
{
int partition = 0;
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("create BrokerRouter ,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri)
{
Log = IntegrationConfig.NoDebugLog
});
stopwatch.Restart();
var producer = new Producer(router)
{
BatchDelayTime = TimeSpan.FromMilliseconds(10), BatchSize = numberOfMessage / 10
};
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("create producer ,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
stopwatch.Restart();
List <OffsetResponse> offsets = await producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic);
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("request Offset,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
stopwatch.Restart();
List <Task> sendList = new List <Task>(numberOfMessage);
for (int i = 0; i < numberOfMessage; i++)
{
var sendTask = producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(i.ToString()) }, 1, null, MessageCodec.CodecNone, partition);
sendList.Add(sendTask);
}
TimeSpan maxTimeToRun = TimeSpan.FromMilliseconds(timeoutInMs);
var doneSend = Task.WhenAll(sendList.ToArray());
await Task.WhenAny(doneSend, Task.Delay(maxTimeToRun));
Assert.IsTrue(doneSend.IsCompleted, "not done to send in time");
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("done send ,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
stopwatch.Restart();
ConsumerOptions consumerOptions = new ConsumerOptions(IntegrationConfig.IntegrationTopic, router);
consumerOptions.PartitionWhitelist = new List <int> {
partition
};
consumerOptions.MaxWaitTimeForMinimumBytes = TimeSpan.Zero;
Consumer consumer = new Consumer(consumerOptions, offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray());
int expected = 0;
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start Consume ,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
IEnumerable <Message> messages = null;
var doneConsume = Task.Run((() =>
{
stopwatch.Restart();
messages = consumer.Consume().Take(numberOfMessage).ToArray();
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("done Consume ,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
stopwatch.Restart();
}));
await Task.WhenAny(doneConsume, Task.Delay(maxTimeToRun));
Assert.IsTrue(doneConsume.IsCompleted, "not done to Consume in time");
Assert.IsTrue(messages.Count() == numberOfMessage, "not Consume all ,messages");
foreach (Message message in messages)
{
Assert.That(message.Value.ToUtf8String(), Is.EqualTo(expected.ToString()),
"Expected the message list in the correct order.");
expected++;
}
stopwatch.Restart();
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start producer Dispose ,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
producer.Dispose();
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start consumer Dispose ,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
consumer.Dispose();
stopwatch.Restart();
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start router Dispose,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
router.Dispose();
}
/// <summary>
/// order Should remain in the same ack leve and partition
/// </summary>
/// <returns></returns>
public async Task ConsumerShouldConsumeInSameOrderAsAsyncProduced()
{
int partition = 0;
int numberOfMessage = 200;
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("create BrokerRouter"));
var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri));
int causesRaceConditionOldVersion = 2;
var producer = new Producer(router, causesRaceConditionOldVersion)
{
BatchDelayTime = TimeSpan.Zero
}; //this is slow on purpose
//this is not slow var producer = new Producer(router);
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("create producer"));
List <OffsetResponse> offsets = await producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic);
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("request Offset"));
List <Task> sendList = new List <Task>(numberOfMessage);
for (int i = 0; i < numberOfMessage; i++)
{
var sendTask = producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(i.ToString()) }, 1, null, MessageCodec.CodecNone, partition);
sendList.Add(sendTask);
}
await Task.WhenAll(sendList.ToArray());
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("done send"));
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("create Consumer"));
ConsumerOptions consumerOptions = new ConsumerOptions(IntegrationConfig.IntegrationTopic, router);
consumerOptions.PartitionWhitelist = new List <int> {
partition
};
Consumer consumer = new Consumer(consumerOptions, offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray());
int expected = 0;
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start Consume"));
await Task.Run((() =>
{
var results = consumer.Consume().Take(numberOfMessage).ToList();
Assert.IsTrue(results.Count() == numberOfMessage, "not Consume all ,messages");
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("done Consume"));
foreach (Message message in results)
{
Assert.That(message.Value.ToUtf8String(), Is.EqualTo(expected.ToString()),
"Expected the message list in the correct order.");
expected++;
}
}));
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start producer Dispose"));
producer.Dispose();
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start consumer Dispose"));
consumer.Dispose();
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start router Dispose"));
router.Dispose();
}
static void Main(string[] args)
{
var options = new KafkaOptions(new Uri("http://kafka1:9092"), new Uri("http://kafka2:9092"))
{
Log = new ConsoleLog()
};
var router = new BrokerRouter(options);
var client = new Producer(router);
var timing = new RollingQueue <double>(50);
var rate = new RollingQueue <double>(50);
var second = DateTime.Now.Second;
var count = 0;
Task.Run(() =>
{
var consumer = new Consumer(new ConsumerOptions("latencies", router));
var position = consumer.GetTopicOffsetAsync("latencies");
position.Wait();
consumer.SetOffsetPosition(
position.Result
.Select(p => new OffsetPosition(p.PartitionId, p.Offsets.First()))
.ToArray()
);
foreach (var data in consumer.Consume())
{
count++;
var rtt = (DateTime.Now - new DateTime(
long.Parse(Encoding.UTF8.GetString(data.Value))
)).TotalMilliseconds;
if (rtt < 1000)
{
timing.Enqueue(rtt);
}
if (second != DateTime.Now.Second)
{
second = DateTime.Now.Second;
rate.Enqueue(count);
count = 0;
Console.WriteLine("Rate: {0} pps.\t{1} ",
rate.Average().ToString("N2"), (rtt < 1000)
? "RTT: " + timing.Average().ToString("N2") + " ms."
: string.Empty
);
}
}
});
while (true)
{
client.SendMessageAsync("latencies", new[] {
new Message(DateTime.Now.Ticks.ToString()),
new Message(DateTime.Now.Ticks.ToString()),
new Message(DateTime.Now.Ticks.ToString()),
new Message(DateTime.Now.Ticks.ToString()),
new Message(DateTime.Now.Ticks.ToString()),
new Message(DateTime.Now.Ticks.ToString()),
new Message(DateTime.Now.Ticks.ToString()),
new Message(DateTime.Now.Ticks.ToString()),
new Message(DateTime.Now.Ticks.ToString()),
new Message(DateTime.Now.Ticks.ToString())
}, 1);
Thread.Sleep(1);
}
client.Dispose();
router.Dispose();
}