public static void WatermarkOffsets(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start WatermarkOffsets");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var testString = "hello world";
DeliveryReport <Null, string> dr;
using (var producer = new Producer <Null, string>(producerConfig))
using (var adminClient = new AdminClient(producer.Handle))
{
dr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> {
Value = testString
}).Result;
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); // this isn't necessary.
var queryOffsets = adminClient.QueryWatermarkOffsets(new TopicPartition(singlePartitionTopic, 0), TimeSpan.FromSeconds(20));
Assert.NotEqual(queryOffsets.Low, Offset.Invalid);
Assert.NotEqual(queryOffsets.High, Offset.Invalid);
// TODO: can anything be said about the high watermark offset c.f. dr.Offset?
// I have seen queryOffsets.High < dr.Offset and also queryOffsets.High = dr.Offset + 1.
// The former only once (or was I in error?). request.required.acks has a default value
// of 1, so with only one broker, I assume the former should never happen.
// Console.WriteLine($"Query Offsets: [{queryOffsets.Low} {queryOffsets.High}]. DR Offset: {dr.Offset}");
Assert.True(queryOffsets.Low < queryOffsets.High);
}
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000
};
using (var consumer = new Consumer <byte[], byte[]>(consumerConfig))
using (var adminClient = new AdminClient(consumer.Handle))
{
consumer.Assign(new List <TopicPartitionOffset>()
{
dr.TopicPartitionOffset
});
var record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
var getOffsets = adminClient.GetWatermarkOffsets(dr.TopicPartition);
Assert.Equal(getOffsets.Low, Offset.Invalid);
// the offset of the next message to be read.
Assert.Equal(getOffsets.High, dr.Offset + 1);
var queryOffsets = adminClient.QueryWatermarkOffsets(dr.TopicPartition, TimeSpan.FromSeconds(20));
Assert.NotEqual(queryOffsets.Low, Offset.Invalid);
Assert.Equal(getOffsets.High, queryOffsets.High);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end WatermarkOffsets");
}
public static void Producer_Handles(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start Producer_Handles");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
using (var producer1 = new ProducerBuilder <byte[], byte[]>(producerConfig).Build())
using (var producer2 = new DependentProducerBuilder <string, string>(producer1.Handle).Build())
using (var producer3 = new DependentProducerBuilder <byte[], byte[]>(producer1.Handle).Build())
using (var producer4 = new DependentProducerBuilder <int, string>(producer2.Handle).Build())
using (var producer5 = new DependentProducerBuilder <int, int>(producer3.Handle).Build())
using (var producer6 = new DependentProducerBuilder <string, byte[]>(producer4.Handle).Build())
using (var producer7 = new ProducerBuilder <double, double>(producerConfig).Build())
using (var adminClient = new AdminClient(producer7.Handle))
{
var r1 = producer1.ProduceAsync(topic.Name, new Message <byte[], byte[]> {
Key = new byte[] { 42 }, Value = new byte[] { 33 }
}).Result;
Assert.Equal(new byte[] { 42 }, r1.Key);
Assert.Equal(new byte[] { 33 }, r1.Value);
Assert.Equal(0, r1.Offset);
var r2 = producer2.ProduceAsync(topic.Name, new Message <string, string> {
Key = "hello", Value = "world"
}).Result;
Assert.Equal("hello", r2.Key);
Assert.Equal("world", r2.Value);
var r3 = producer3.ProduceAsync(topic.Name, new Message <byte[], byte[]> {
Key = new byte[] { 40 }, Value = new byte[] { 31 }
}).Result;
Assert.Equal(new byte[] { 40 }, r3.Key);
Assert.Equal(new byte[] { 31 }, r3.Value);
var r4 = producer4.ProduceAsync(topic.Name, new Message <int, string> {
Key = 42, Value = "mellow world"
}).Result;
Assert.Equal(42, r4.Key);
Assert.Equal("mellow world", r4.Value);
var r5 = producer5.ProduceAsync(topic.Name, new Message <int, int> {
Key = int.MaxValue, Value = int.MinValue
}).Result;
Assert.Equal(int.MaxValue, r5.Key);
Assert.Equal(int.MinValue, r5.Value);
var r6 = producer6.ProduceAsync(topic.Name, new Message <string, byte[]> {
Key = "yellow mould", Value = new byte[] { 69 }
}).Result;
Assert.Equal("yellow mould", r6.Key);
Assert.Equal(new byte[] { 69 }, r6.Value);
var r7 = producer7.ProduceAsync(topic.Name, new Message <double, double> {
Key = 44.0, Value = 234.4
}).Result;
Assert.Equal(44.0, r7.Key);
Assert.Equal(234.4, r7.Value);
var offsets = adminClient.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(10));
Assert.Equal(0, offsets.Low);
Assert.Equal(7, offsets.High);
// implicitly check this does not throw.
}
var consumerConfig = new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = Guid.NewGuid().ToString()
};
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
var r1 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(new byte[] { 42 }, r1.Key);
Assert.Equal(new byte[] { 33 }, r1.Value);
Assert.Equal(0, r1.Offset);
}
using (var consumer = new ConsumerBuilder <string, string>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 1));
var r2 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal("hello", r2.Key);
Assert.Equal("world", r2.Value);
Assert.Equal(1, r2.Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 2));
var r3 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(new byte[] { 40 }, r3.Key);
Assert.Equal(new byte[] { 31 }, r3.Value);
Assert.Equal(2, r3.Offset);
}
using (var consumer = new ConsumerBuilder <int, string>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 3));
var r4 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(42, r4.Key);
Assert.Equal("mellow world", r4.Value);
Assert.Equal(3, r4.Offset);
}
using (var consumer = new ConsumerBuilder <int, int>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 4));
var r5 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(int.MaxValue, r5.Key);
Assert.Equal(int.MinValue, r5.Value);
Assert.Equal(4, r5.Offset);
}
using (var consumer = new ConsumerBuilder <string, byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 5));
var r6 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal("yellow mould", r6.Key);
Assert.Equal(new byte[] { 69 }, r6.Value);
Assert.Equal(5, r6.Offset);
}
using (var consumer = new ConsumerBuilder <double, double>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 6));
var r7 = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.Equal(44.0, r7.Key);
Assert.Equal(234.4, r7.Value);
Assert.Equal(6, r7.Offset);
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Producer_Handles");
}
