public async Task EnsureGzipCompressedMessageCanSend()
{
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start EnsureGzipCompressedMessageCanSend"));
using (var conn = GetKafkaConnection())
{
conn.SendAsync(new MetadataRequest
{
Topics = new List <string>(new[] { IntegrationConfig.IntegrationCompressionTopic })
})
.Wait(TimeSpan.FromSeconds(10));
}
using (var router = new BrokerRouter(_options))
{
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start RefreshMissingTopicMetadata"));
await router.RefreshMissingTopicMetadata(IntegrationConfig.IntegrationCompressionTopic);
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("end RefreshMissingTopicMetadata"));
var conn = router.SelectBrokerRouteFromLocalCache(IntegrationConfig.IntegrationCompressionTopic, 0);
var request = new ProduceRequest
{
Acks = 1,
TimeoutMS = 1000,
Payload = new List <Payload>
{
new Payload
{
Codec = MessageCodec.CodecGzip,
Topic = IntegrationConfig.IntegrationCompressionTopic,
Partition = 0,
Messages = new List <Message>
{
new Message("0", "1"),
new Message("1", "1"),
new Message("2", "1")
}
}
}
};
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start SendAsync"));
var response = conn.Connection.SendAsync(request).Result;
IntegrationConfig.NoDebugLog.InfoFormat("end SendAsync");
Assert.That(response.First().Error, Is.EqualTo(0));
IntegrationConfig.NoDebugLog.InfoFormat("start dispose");
}
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("end EnsureGzipCompressedMessageCanSend"));
}
public void ConsumerShouldBeAbleToSeekBackToEarlierOffset()
{
var expected = new List <string> {
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19"
};
var testId = Guid.NewGuid().ToString();
using (var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri)))
using (var producer = new Producer(router))
{
var offsets = producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic).Result
.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray();
using (var consumer = new Consumer(new ConsumerOptions(IntegrationConfig.IntegrationTopic, router)
{
MaxWaitTimeForMinimumBytes = TimeSpan.Zero
}, offsets))
{
for (int i = 0; i < 20; i++)
{
producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(i.ToString(), testId) }).Wait();
}
var sentMessages = consumer.Consume().Take(20).ToList();
//ensure the produced messages arrived
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("Message order: {0}", string.Join(", ", sentMessages.Select(x => x.Value.ToUtf8String()).ToList())));
Assert.That(sentMessages.Count, Is.EqualTo(20));
Assert.That(sentMessages.Select(x => x.Value.ToUtf8String()).ToList(), Is.EqualTo(expected));
Assert.That(sentMessages.Any(x => x.Key.ToUtf8String() != testId), Is.False);
//seek back to initial offset
consumer.SetOffsetPosition(offsets);
var resetPositionMessages = consumer.Consume().Take(20).ToList();
//ensure all produced messages arrive again
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("Message order: {0}", string.Join(", ", resetPositionMessages.Select(x => x.Value).ToList())));
Assert.That(resetPositionMessages.Count, Is.EqualTo(20));
Assert.That(resetPositionMessages.Select(x => x.Value.ToUtf8String()).ToList(), Is.EqualTo(expected));
Assert.That(resetPositionMessages.Any(x => x.Key.ToUtf8String() != testId), Is.False);
}
}
}
public async Task ConsumerShouldConsumeInSameOrderAsAsyncProduced_dataLoad(int numberOfMessage, int timeoutInMs)
{
int partition = 0;
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("create BrokerRouter ,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri)
{
Log = IntegrationConfig.NoDebugLog
});
stopwatch.Restart();
var producer = new Producer(router)
{
BatchDelayTime = TimeSpan.FromMilliseconds(10), BatchSize = numberOfMessage / 10
};
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("create producer ,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
stopwatch.Restart();
List <OffsetResponse> offsets = await producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic);
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("request Offset,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
stopwatch.Restart();
List <Task> sendList = new List <Task>(numberOfMessage);
for (int i = 0; i < numberOfMessage; i++)
{
var sendTask = producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(i.ToString()) }, 1, null, MessageCodec.CodecNone, partition);
sendList.Add(sendTask);
}
TimeSpan maxTimeToRun = TimeSpan.FromMilliseconds(timeoutInMs);
var doneSend = Task.WhenAll(sendList.ToArray());
await Task.WhenAny(doneSend, Task.Delay(maxTimeToRun));
Assert.IsTrue(doneSend.IsCompleted, "not done to send in time");
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("done send ,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
stopwatch.Restart();
ConsumerOptions consumerOptions = new ConsumerOptions(IntegrationConfig.IntegrationTopic, router);
consumerOptions.PartitionWhitelist = new List <int> {
partition
};
consumerOptions.MaxWaitTimeForMinimumBytes = TimeSpan.Zero;
Consumer consumer = new Consumer(consumerOptions, offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray());
int expected = 0;
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start Consume ,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
IEnumerable <Message> messages = null;
var doneConsume = Task.Run((() =>
{
stopwatch.Restart();
messages = consumer.Consume().Take(numberOfMessage).ToArray();
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("done Consume ,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
stopwatch.Restart();
}));
await Task.WhenAny(doneConsume, Task.Delay(maxTimeToRun));
Assert.IsTrue(doneConsume.IsCompleted, "not done to Consume in time");
Assert.IsTrue(messages.Count() == numberOfMessage, "not Consume all ,messages");
foreach (Message message in messages)
{
Assert.That(message.Value.ToUtf8String(), Is.EqualTo(expected.ToString()),
"Expected the message list in the correct order.");
expected++;
}
stopwatch.Restart();
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start producer Dispose ,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
producer.Dispose();
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start consumer Dispose ,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
consumer.Dispose();
stopwatch.Restart();
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start router Dispose,time Milliseconds:{0}", stopwatch.ElapsedMilliseconds));
router.Dispose();
}
/// <summary>
/// order Should remain in the same ack leve and partition
/// </summary>
/// <returns></returns>
public async Task ConsumerShouldConsumeInSameOrderAsAsyncProduced()
{
int partition = 0;
int numberOfMessage = 200;
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("create BrokerRouter"));
var router = new BrokerRouter(new KafkaOptions(IntegrationConfig.IntegrationUri));
int causesRaceConditionOldVersion = 2;
var producer = new Producer(router, causesRaceConditionOldVersion)
{
BatchDelayTime = TimeSpan.Zero
}; //this is slow on purpose
//this is not slow var producer = new Producer(router);
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("create producer"));
List <OffsetResponse> offsets = await producer.GetTopicOffsetAsync(IntegrationConfig.IntegrationTopic);
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("request Offset"));
List <Task> sendList = new List <Task>(numberOfMessage);
for (int i = 0; i < numberOfMessage; i++)
{
var sendTask = producer.SendMessageAsync(IntegrationConfig.IntegrationTopic, new[] { new Message(i.ToString()) }, 1, null, MessageCodec.CodecNone, partition);
sendList.Add(sendTask);
}
await Task.WhenAll(sendList.ToArray());
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("done send"));
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("create Consumer"));
ConsumerOptions consumerOptions = new ConsumerOptions(IntegrationConfig.IntegrationTopic, router);
consumerOptions.PartitionWhitelist = new List <int> {
partition
};
Consumer consumer = new Consumer(consumerOptions, offsets.Select(x => new OffsetPosition(x.PartitionId, x.Offsets.Max())).ToArray());
int expected = 0;
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start Consume"));
await Task.Run((() =>
{
var results = consumer.Consume().Take(numberOfMessage).ToList();
Assert.IsTrue(results.Count() == numberOfMessage, "not Consume all ,messages");
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("done Consume"));
foreach (Message message in results)
{
Assert.That(message.Value.ToUtf8String(), Is.EqualTo(expected.ToString()),
"Expected the message list in the correct order.");
expected++;
}
}));
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start producer Dispose"));
producer.Dispose();
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start consumer Dispose"));
consumer.Dispose();
IntegrationConfig.NoDebugLog.InfoFormat(IntegrationConfig.Highlight("start router Dispose"));
router.Dispose();
}