public static WatermarkOffsets BuscarOffsetsDisponiveis(this ConsumidorAbstrato ob, string topico, string consumerGroup)
{
ob.ConsumerConfig.GroupId = consumerGroup;
using (var consumer = new ConsumerBuilder <string, string>(ob.ConsumerConfig).Build())
{
consumer.Subscribe(topico);
var assignment = consumer.Assignment;
while (assignment.Count == 0)
{
Thread.Sleep(50); //ha um bug no driver para .net, as vezes é preciso aguardar a informação chegar
assignment = consumer.Assignment;
}
var topicPartition = new TopicPartition(assignment[0].Topic, assignment[0].Partition);
var wmo = consumer.GetWatermarkOffsets(topicPartition);
while (wmo.High.Value == -1001) //ha um bug no driver para .net, as vezes é preciso aguardar a informação chegar
{
wmo = consumer.GetWatermarkOffsets(topicPartition);
Thread.Sleep(50);
}
return(wmo);
}
}
public void WatermarkOffsets(string bootstrapServers)
{
LogToFile("start WatermarkOffsets");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var testString = "hello world";
DeliveryResult <Null, string> dr;
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
{
dr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> {
Value = testString
}).Result;
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); // this isn't necessary.
}
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000
};
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new List <TopicPartitionOffset>()
{
dr.TopicPartitionOffset
});
var record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
var getOffsets = consumer.GetWatermarkOffsets(dr.TopicPartition);
Assert.Equal(getOffsets.Low, Offset.Unset);
// the offset of the next message to be read.
Assert.Equal(getOffsets.High, dr.Offset + 1);
var queryOffsets = consumer.QueryWatermarkOffsets(dr.TopicPartition, TimeSpan.FromSeconds(20));
Assert.NotEqual(queryOffsets.Low, Offset.Unset);
Assert.Equal(getOffsets.High, queryOffsets.High);
}
// Test empty topic case
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
var wo = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(30));
// Refer to WatermarkOffsets class documentation for more information.
Assert.Equal(0, wo.Low);
Assert.Equal(0, wo.High);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end WatermarkOffsets");
}
public static void WatermarkOffsets(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start WatermarkOffsets");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var testString = "hello world";
DeliveryResult <Null, string> dr;
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
using (var adminClient = new AdminClient(producer.Handle))
{
dr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> {
Value = testString
}).Result;
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); // this isn't necessary.
}
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000
};
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new List <TopicPartitionOffset>()
{
dr.TopicPartitionOffset
});
var record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
var getOffsets = consumer.GetWatermarkOffsets(dr.TopicPartition);
Assert.Equal(getOffsets.Low, Offset.Invalid);
// the offset of the next message to be read.
Assert.Equal(getOffsets.High, dr.Offset + 1);
var queryOffsets = consumer.QueryWatermarkOffsets(dr.TopicPartition, TimeSpan.FromSeconds(20));
Assert.NotEqual(queryOffsets.Low, Offset.Invalid);
Assert.Equal(getOffsets.High, queryOffsets.High);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end WatermarkOffsets");
}
static internal async Task Start(string[] args)
{
// var config = new ProducerConfig
// {
// BootstrapServers = "kafka:9092",
// MessageTimeoutMs = 30000.
// ApiVersionRequest = false,
// };
// // If serializers are not specified, default serializers from
// // `Confluent.Kafka.Serializers` will be automatically used where
// // available. Note: by default strings are encoded as UTF8.
// using (var p = new ProducerBuilder<Null, string>(config).Build())
// {
// try
// {
// var dr = await p.ProduceAsync("test-topic", new Message<Null, string> { Value = "tes2t" });
// Console.WriteLine($"Delivered '{dr.Value}' to '{dr.TopicPartitionOffset}'");
// }
// catch (ProduceException<Null, string> e)
// {
// Console.WriteLine($"Delivery failed: {e.Error.Reason}");
// }
// }
var config = new ConsumerConfig
{
BootstrapServers = "kafka:9092",
AutoOffsetReset = AutoOffsetReset.Earliest,
GroupId = "hz",
EnableAutoCommit = false,
ApiVersionRequest = false,
};
// If serializers are not specified, default serializers from
// `Confluent.Kafka.Serializers` will be automatically used where
// available. Note: by default strings are encoded as UTF8.
using (var c = new ConsumerBuilder <Null, string>(config).Build())
{
try
{
c.Subscribe("test-topic");
var o = c.Position(new TopicPartition("test-topic", new Partition(0)));
var a = c.GetWatermarkOffsets(new TopicPartition("test-topic", new Partition(0)));
var cr = c.Consume();
}
catch (ConsumeException e)
{
}
}
}
public void Transactions_WatermarkOffsets(string bootstrapServers)
{
LogToFile("start Transactions_WatermarkOffsets");
var groupName = Guid.NewGuid().ToString();
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString(), LingerMs = 0
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
IsolationLevel = IsolationLevel.ReadCommitted, BootstrapServers = bootstrapServers, GroupId = groupName, EnableAutoCommit = false
}).Build())
{
var wo1 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
Assert.Equal(Offset.Unset, wo1.Low);
Assert.Equal(Offset.Unset, wo1.High);
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
producer.InitTransactions(TimeSpan.FromSeconds(30));
producer.BeginTransaction();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message1"
}).Wait();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message2"
}).Wait();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message3"
}).Wait();
WatermarkOffsets wo2 = new WatermarkOffsets(Offset.Unset, Offset.Unset);
for (int i = 0; i < 10; ++i)
{
var cr = consumer.Consume(TimeSpan.FromMilliseconds(500));
wo2 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
if (wo2.High == 3)
{
break;
}
}
Assert.Equal(3, wo2.High);
producer.CommitTransaction(TimeSpan.FromSeconds(30));
WatermarkOffsets wo3 = new WatermarkOffsets(Offset.Unset, Offset.Unset);
for (int i = 0; i < 10; ++i)
{
var cr2 = consumer.Consume(TimeSpan.FromSeconds(500));
wo3 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
if (wo3.High > 3)
{
break;
}
}
Assert.Equal(4, wo3.High);
var wo4 = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(30));
Assert.Equal(4, wo4.High);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_WatermarkOffsets");
}