public static WatermarkOffsets BuscarOffsetsDisponiveis(this ConsumidorAbstrato ob, string topico, string consumerGroup) { ob.ConsumerConfig.GroupId = consumerGroup; using (var consumer = new ConsumerBuilder <string, string>(ob.ConsumerConfig).Build()) { consumer.Subscribe(topico); var assignment = consumer.Assignment; while (assignment.Count == 0) { Thread.Sleep(50); //ha um bug no driver para .net, as vezes é preciso aguardar a informação chegar assignment = consumer.Assignment; } var topicPartition = new TopicPartition(assignment[0].Topic, assignment[0].Partition); var wmo = consumer.GetWatermarkOffsets(topicPartition); while (wmo.High.Value == -1001) //ha um bug no driver para .net, as vezes é preciso aguardar a informação chegar { wmo = consumer.GetWatermarkOffsets(topicPartition); Thread.Sleep(50); } return(wmo); } }
public void WatermarkOffsets(string bootstrapServers) { LogToFile("start WatermarkOffsets"); var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var testString = "hello world"; DeliveryResult <Null, string> dr; using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build()) { dr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> { Value = testString }).Result; Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); // this isn't necessary. } var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, SessionTimeoutMs = 6000 }; using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new List <TopicPartitionOffset>() { dr.TopicPartitionOffset }); var record = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.NotNull(record.Message); var getOffsets = consumer.GetWatermarkOffsets(dr.TopicPartition); Assert.Equal(getOffsets.Low, Offset.Unset); // the offset of the next message to be read. Assert.Equal(getOffsets.High, dr.Offset + 1); var queryOffsets = consumer.QueryWatermarkOffsets(dr.TopicPartition, TimeSpan.FromSeconds(20)); Assert.NotEqual(queryOffsets.Low, Offset.Unset); Assert.Equal(getOffsets.High, queryOffsets.High); } // Test empty topic case using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { var wo = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(30)); // Refer to WatermarkOffsets class documentation for more information. Assert.Equal(0, wo.Low); Assert.Equal(0, wo.High); } Assert.Equal(0, Library.HandleCount); LogToFile("end WatermarkOffsets"); }
public static void WatermarkOffsets(string bootstrapServers, string singlePartitionTopic, string partitionedTopic) { LogToFile("start WatermarkOffsets"); var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var testString = "hello world"; DeliveryResult <Null, string> dr; using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build()) using (var adminClient = new AdminClient(producer.Handle)) { dr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> { Value = testString }).Result; Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); // this isn't necessary. } var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, SessionTimeoutMs = 6000 }; using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new List <TopicPartitionOffset>() { dr.TopicPartitionOffset }); var record = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.NotNull(record.Message); var getOffsets = consumer.GetWatermarkOffsets(dr.TopicPartition); Assert.Equal(getOffsets.Low, Offset.Invalid); // the offset of the next message to be read. Assert.Equal(getOffsets.High, dr.Offset + 1); var queryOffsets = consumer.QueryWatermarkOffsets(dr.TopicPartition, TimeSpan.FromSeconds(20)); Assert.NotEqual(queryOffsets.Low, Offset.Invalid); Assert.Equal(getOffsets.High, queryOffsets.High); } Assert.Equal(0, Library.HandleCount); LogToFile("end WatermarkOffsets"); }
static internal async Task Start(string[] args) { // var config = new ProducerConfig // { // BootstrapServers = "kafka:9092", // MessageTimeoutMs = 30000. // ApiVersionRequest = false, // }; // // If serializers are not specified, default serializers from // // `Confluent.Kafka.Serializers` will be automatically used where // // available. Note: by default strings are encoded as UTF8. // using (var p = new ProducerBuilder<Null, string>(config).Build()) // { // try // { // var dr = await p.ProduceAsync("test-topic", new Message<Null, string> { Value = "tes2t" }); // Console.WriteLine($"Delivered '{dr.Value}' to '{dr.TopicPartitionOffset}'"); // } // catch (ProduceException<Null, string> e) // { // Console.WriteLine($"Delivery failed: {e.Error.Reason}"); // } // } var config = new ConsumerConfig { BootstrapServers = "kafka:9092", AutoOffsetReset = AutoOffsetReset.Earliest, GroupId = "hz", EnableAutoCommit = false, ApiVersionRequest = false, }; // If serializers are not specified, default serializers from // `Confluent.Kafka.Serializers` will be automatically used where // available. Note: by default strings are encoded as UTF8. using (var c = new ConsumerBuilder <Null, string>(config).Build()) { try { c.Subscribe("test-topic"); var o = c.Position(new TopicPartition("test-topic", new Partition(0))); var a = c.GetWatermarkOffsets(new TopicPartition("test-topic", new Partition(0))); var cr = c.Consume(); } catch (ConsumeException e) { } } }
public void Transactions_WatermarkOffsets(string bootstrapServers) { LogToFile("start Transactions_WatermarkOffsets"); var groupName = Guid.NewGuid().ToString(); using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var producer = new ProducerBuilder <string, string>(new ProducerConfig { BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString(), LingerMs = 0 }).Build()) using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig { IsolationLevel = IsolationLevel.ReadCommitted, BootstrapServers = bootstrapServers, GroupId = groupName, EnableAutoCommit = false }).Build()) { var wo1 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0)); Assert.Equal(Offset.Unset, wo1.Low); Assert.Equal(Offset.Unset, wo1.High); consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0)); producer.InitTransactions(TimeSpan.FromSeconds(30)); producer.BeginTransaction(); producer.ProduceAsync(topic.Name, new Message <string, string> { Key = "test", Value = "message1" }).Wait(); producer.ProduceAsync(topic.Name, new Message <string, string> { Key = "test", Value = "message2" }).Wait(); producer.ProduceAsync(topic.Name, new Message <string, string> { Key = "test", Value = "message3" }).Wait(); WatermarkOffsets wo2 = new WatermarkOffsets(Offset.Unset, Offset.Unset); for (int i = 0; i < 10; ++i) { var cr = consumer.Consume(TimeSpan.FromMilliseconds(500)); wo2 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0)); if (wo2.High == 3) { break; } } Assert.Equal(3, wo2.High); producer.CommitTransaction(TimeSpan.FromSeconds(30)); WatermarkOffsets wo3 = new WatermarkOffsets(Offset.Unset, Offset.Unset); for (int i = 0; i < 10; ++i) { var cr2 = consumer.Consume(TimeSpan.FromSeconds(500)); wo3 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0)); if (wo3.High > 3) { break; } } Assert.Equal(4, wo3.High); var wo4 = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(30)); Assert.Equal(4, wo4.High); } Assert.Equal(0, Library.HandleCount); LogToFile("end Transactions_WatermarkOffsets"); }