public static void Consumer_Seek(string bootstrapServers, string singlePartitionTopic, string partitionedTopic) { LogToFile("start Consumer_Seek"); var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers }; var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; using (var producer = new ProducerBuilder <byte[], byte[]>(producerConfig).Build()) using (var consumer = new ConsumerBuilder <Null, string>(consumerConfig) .SetErrorHandler((_, e) => Assert.True(false, e.Reason)) .Build()) { const string checkValue = "check value"; var dr = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> { Value = Serializers.Utf8.Serialize(checkValue, true, null, null) }).Result; var dr2 = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> { Value = Serializers.Utf8.Serialize("second value", true, null, null) }).Result; var dr3 = producer.ProduceAsync(singlePartitionTopic, new Message <byte[], byte[]> { Value = Serializers.Utf8.Serialize("third value", true, null, null) }).Result; consumer.Assign(new TopicPartitionOffset[] { new TopicPartitionOffset(singlePartitionTopic, 0, dr.Offset) }); var record = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.NotNull(record.Message); record = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.NotNull(record.Message); record = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.NotNull(record.Message); consumer.Seek(dr.TopicPartitionOffset); // position is that of the last consumed offset. it shouldn't be equal to the seek position. var pos = consumer.Position(new List <TopicPartition> { dr.TopicPartition }).First(); Assert.NotEqual(dr.Offset, pos.Offset); record = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.NotNull(record.Message); Assert.Equal(checkValue, record.Message.Value); } Assert.Equal(0, Library.HandleCount); LogToFile("end Consumer_Seek"); }
public static void Consume() { var conf = new ConsumerConfig { GroupId = "g3", BootstrapServers = "localhost:9092", // Note: The AutoOffsetReset property determines the start offset in the event // there are not yet any committed offsets for the consumer group for the // topic/partitions of interest. By default, offsets are committed // automatically, so in this example, consumption will only start from the // earliest message in the topic 'my-topic' the first time you run the program. AutoOffsetReset = AutoOffsetReset.Earliest }; using (var c = new ConsumerBuilder <Ignore, string>(conf).Build()) { c.Subscribe("Topic2"); var listTopicPartition = c.Assignment; foreach (var topicPartition in c.Assignment) { var offset = c.Position(topicPartition); } System.Threading.CancellationTokenSource cts = new System.Threading.CancellationTokenSource(); Console.CancelKeyPress += (_, e) => { e.Cancel = true; // prevent the process from terminating. cts.Cancel(); }; try { while (true) { try { var cr = c.Consume(cts.Token); Console.WriteLine($"Consumed message '{cr.Message?.Value}' at: '{cr.TopicPartitionOffset}'."); UpdateValue(); } catch (ConsumeException e) { Console.WriteLine($"Error occured: {e.Error.Reason}"); } } } catch (OperationCanceledException) { // Ensure the consumer leaves the group cleanly and final offsets are committed. c.Close(); } } }
static internal async Task Start(string[] args) { // var config = new ProducerConfig // { // BootstrapServers = "kafka:9092", // MessageTimeoutMs = 30000. // ApiVersionRequest = false, // }; // // If serializers are not specified, default serializers from // // `Confluent.Kafka.Serializers` will be automatically used where // // available. Note: by default strings are encoded as UTF8. // using (var p = new ProducerBuilder<Null, string>(config).Build()) // { // try // { // var dr = await p.ProduceAsync("test-topic", new Message<Null, string> { Value = "tes2t" }); // Console.WriteLine($"Delivered '{dr.Value}' to '{dr.TopicPartitionOffset}'"); // } // catch (ProduceException<Null, string> e) // { // Console.WriteLine($"Delivery failed: {e.Error.Reason}"); // } // } var config = new ConsumerConfig { BootstrapServers = "kafka:9092", AutoOffsetReset = AutoOffsetReset.Earliest, GroupId = "hz", EnableAutoCommit = false, ApiVersionRequest = false, }; // If serializers are not specified, default serializers from // `Confluent.Kafka.Serializers` will be automatically used where // available. Note: by default strings are encoded as UTF8. using (var c = new ConsumerBuilder <Null, string>(config).Build()) { try { c.Subscribe("test-topic"); var o = c.Position(new TopicPartition("test-topic", new Partition(0))); var a = c.GetWatermarkOffsets(new TopicPartition("test-topic", new Partition(0))); var cr = c.Consume(); } catch (ConsumeException e) { } } }
public void Consumer_Commit_Committed_Position(string bootstrapServers) { LogToFile("start Consumer_Commit_Committed_Position"); const int N = 8; const int Partition = 0; var messages = ProduceMessages(bootstrapServers, singlePartitionTopic, Partition, N); var firstMsgOffset = messages[0].Offset; var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, EnableAutoCommit = false }; var firstMessage = messages[0]; var lastMessage = messages[N - 1]; using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset)); // Test #0 (empty cases) consumer.Commit(new List <TopicPartitionOffset>()); // should not throw. var committed = consumer.Committed(new List <TopicPartition>(), TimeSpan.FromSeconds(10)); Assert.Empty(committed); var ps = consumer.Position(new List <TopicPartition>()); Assert.Empty(ps); // Test #0.5 (invalid cases) ps = consumer.Position(new List <TopicPartition> { new TopicPartition("invalid-topic", 0) }); Assert.Single(ps); Assert.Equal(Offset.Invalid, ps[0].Offset); Assert.Equal("invalid-topic", ps[0].Topic); Assert.Equal(0, (int)ps[0].Partition); Assert.Equal(new TopicPartition("invalid-topic", 0), ps[0].TopicPartition); // Test #1 var record = consumer.Consume(TimeSpan.FromMilliseconds(6000)); var os = consumer.Commit(); Assert.Equal(firstMsgOffset + 1, os[0].Offset); ps = consumer.Position(new List <TopicPartition> { new TopicPartition(singlePartitionTopic, 0) }); var co = consumer.Committed(new List <TopicPartition> { new TopicPartition(singlePartitionTopic, 0) }, TimeSpan.FromSeconds(10)); Assert.Equal(firstMsgOffset + 1, co[0].Offset); Assert.Equal(firstMsgOffset + 1, ps[0].Offset); // Test #2 var record2 = consumer.Consume(TimeSpan.FromMilliseconds(6000)); os = consumer.Commit(); Assert.Equal(firstMsgOffset + 2, os[0].Offset); ps = consumer.Position(new List <TopicPartition> { new TopicPartition(singlePartitionTopic, 0) }); co = consumer.Committed(new List <TopicPartition> { new TopicPartition(singlePartitionTopic, 0) }, TimeSpan.FromSeconds(10)); Assert.Equal(firstMsgOffset + 2, ps[0].Offset); Assert.Equal(firstMsgOffset + 2, ps[0].Offset); } // Test #3 using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Commit(new List <TopicPartitionOffset> { new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset + 5) }); var co = consumer.Committed(new List <TopicPartition> { new TopicPartition(singlePartitionTopic, 0) }, TimeSpan.FromSeconds(10)); Assert.Equal(firstMsgOffset + 5, co[0].Offset); } using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new TopicPartition(singlePartitionTopic, 0)); var record = consumer.Consume(TimeSpan.FromMilliseconds(6000)); var ps = consumer.Position(new List <TopicPartition> { new TopicPartition(singlePartitionTopic, 0) }); Assert.Equal(firstMsgOffset + 6, ps[0].Offset); var co = consumer.Committed(new List <TopicPartition> { new TopicPartition(singlePartitionTopic, 0) }, TimeSpan.FromSeconds(10)); Assert.Equal(firstMsgOffset + 5, co[0].Offset); } // Test #4 using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new TopicPartition(singlePartitionTopic, 0)); consumer.Commit(new List <TopicPartitionOffset> { new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset + 3) }); var co = consumer.Committed(new List <TopicPartition> { new TopicPartition(singlePartitionTopic, 0) }, TimeSpan.FromSeconds(10)); Assert.Equal(firstMsgOffset + 3, co[0].Offset); } using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new TopicPartition(singlePartitionTopic, 0)); var record = consumer.Consume(TimeSpan.FromMilliseconds(6000)); var ps = consumer.Position(new List <TopicPartition> { new TopicPartition(singlePartitionTopic, 0) }); Assert.Equal(firstMsgOffset + 4, ps[0].Offset); var co = consumer.Committed(new List <TopicPartition> { new TopicPartition(singlePartitionTopic, 0) }, TimeSpan.FromSeconds(10)); Assert.Equal(firstMsgOffset + 3, co[0].Offset); } // Test #5 using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset)); var record = consumer.Consume(TimeSpan.FromMilliseconds(6000)); var record2 = consumer.Consume(TimeSpan.FromMilliseconds(6000)); var record3 = consumer.Consume(TimeSpan.FromMilliseconds(6000)); consumer.Commit(record3); var record4 = consumer.Consume(TimeSpan.FromMilliseconds(1000)); var co = consumer.Committed(new List <TopicPartition> { new TopicPartition(singlePartitionTopic, 0) }, TimeSpan.FromSeconds(10)); Assert.Equal(firstMsgOffset + 3, co[0].Offset); } using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new TopicPartition(singlePartitionTopic, 0)); var record = consumer.Consume(TimeSpan.FromMilliseconds(6000)); Assert.Equal(firstMsgOffset + 3, record.Offset); var co = consumer.Committed(new List <TopicPartition> { new TopicPartition(singlePartitionTopic, 0) }, TimeSpan.FromSeconds(10)); Assert.Equal(firstMsgOffset + 3, co[0].Offset); } // Test #6 using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset)); var record = consumer.Consume(TimeSpan.FromMilliseconds(6000)); var record2 = consumer.Consume(TimeSpan.FromMilliseconds(6000)); var record3 = consumer.Consume(TimeSpan.FromMilliseconds(6000)); consumer.Commit(record3); var record4 = consumer.Consume(TimeSpan.FromMilliseconds(6000)); var co = consumer.Committed(new List <TopicPartition> { new TopicPartition(singlePartitionTopic, 0) }, TimeSpan.FromSeconds(10)); Assert.Equal(firstMsgOffset + 3, co[0].Offset); } using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new TopicPartition(singlePartitionTopic, 0)); var record = consumer.Consume(TimeSpan.FromMilliseconds(6000)); Assert.Equal(firstMsgOffset + 3, record.Offset); var co = consumer.Committed(new List <TopicPartition> { new TopicPartition(singlePartitionTopic, 0) }, TimeSpan.FromSeconds(10)); Assert.Equal(firstMsgOffset + 3, co[0].Offset); } Assert.Equal(0, Library.HandleCount); LogToFile("end Consumer_Commit_Committed_Position"); }
/// <summary> /// A transactional (exactly once) processing loop that reads individual words and updates /// the corresponding total count state. /// /// When a rebalance occurs (including on startup), the count state for the incrementally /// assigned partitions is reloaded before the loop commences to update it. For this use- /// case, the CooperativeSticky assignor is much more efficient than the Range or RoundRobin /// assignors since it keeps to a minimum the count state that needs to be materialized. /// </summary> /// <remarks> /// Refer to Processor_MapWords for more detailed comments. /// </remarks> public static void Processor_AggregateWords(string brokerList, string clientId, CancellationToken ct) { if (clientId == null) { throw new Exception("Aggregate words processor requires that a client id is specified."); } var txnCommitPeriod = TimeSpan.FromSeconds(10); var pConfig = new ProducerConfig { BootstrapServers = brokerList, ClientId = clientId + "_producer", TransactionalId = TransactionalIdPrefix_Aggregate + "-" + clientId }; var cConfig = new ConsumerConfig { BootstrapServers = brokerList, ClientId = clientId + "_consumer", GroupId = ConsumerGroup_Aggregate, AutoOffsetReset = AutoOffsetReset.Earliest, // This should be greater than the maximum amount of time required to read in // existing count state. It should not be too large, since a rebalance may be // blocked for this long. MaxPollIntervalMs = 600000, // 10 minutes. EnableAutoCommit = false, // Enable incremental rebalancing by using the CooperativeSticky // assignor (avoid stop-the-world rebalances). This is particularly important, // in the AggregateWords case, since the entire count state for newly assigned // partitions is loaded in the partitions assigned handler. PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky, }; var lastTxnCommit = DateTime.Now; using (var producer = new ProducerBuilder <string, int>(pConfig).Build()) using (var consumer = new ConsumerBuilder <string, Null>(cConfig) .SetPartitionsRevokedHandler((c, partitions) => { var remaining = c.Assignment.Where(tp => partitions.Where(x => x.TopicPartition == tp).Count() == 0); Console.WriteLine( "** AggregateWords consumer group partitions revoked: [" + string.Join(',', partitions.Select(p => p.Partition.Value)) + "], remaining: [" + string.Join(',', remaining.Select(p => p.Partition.Value)) + "]"); // Remove materialized word count state for the partitions that have been revoked. foreach (var tp in partitions) { WordCountState[tp.Partition].Dispose(); WordCountState.Remove(tp.Partition); } producer.SendOffsetsToTransaction( c.Assignment.Select(a => new TopicPartitionOffset(a, c.Position(a))), c.ConsumerGroupMetadata, DefaultTimeout); producer.CommitTransaction(); producer.BeginTransaction(); }) .SetPartitionsLostHandler((c, partitions) => { Console.WriteLine( "** AggregateWords consumer group partitions lost: [" + string.Join(',', partitions.Select(p => p.Partition.Value)) + "]"); // clear materialized word count state for all partitions. foreach (var tp in partitions) { WordCountState[tp.Partition].Dispose(); WordCountState.Remove(tp.Partition); } producer.AbortTransaction(); producer.BeginTransaction(); }) .SetPartitionsAssignedHandler((c, partitions) => { Console.WriteLine( "** AggregateWords consumer group partition assigned: [" + string.Join(',', partitions.Select(p => p.Partition.Value)) + "], all: [" + string.Join(',', c.Assignment.Concat(partitions).Select(p => p.Partition.Value)) + "]"); if (partitions.Count > 0) { // Initialize FASTER KV stores for each new partition. foreach (var tp in partitions) { var partitionState = new FasterState(tp.Partition); WordCountState.Add(tp.Partition, partitionState); } // Materialize count state for partitions into the FASTER KV stores. // Note: the partiioning of Topic_Counts matches Topic_Words. LoadCountState(brokerList, partitions.Select(tp => tp.Partition), ct); } }) .Build()) { consumer.Subscribe(Topic_Words); producer.InitTransactions(DefaultTimeout); producer.BeginTransaction(); var wCount = 0; while (true) { try { ct.ThrowIfCancellationRequested(); var cr = consumer.Consume(TimeSpan.FromSeconds(1)); if (cr != null) { string key = cr.Message.Key; var(_, count) = WordCountState[cr.Partition].Session.Read(cr.Message.Key); count += 1; WordCountState[cr.Partition].Session.Upsert(key, count); producer.Produce(Topic_Counts, new Message <string, int> { Key = cr.Message.Key, Value = count }); wCount += 1; } if (DateTime.Now > lastTxnCommit + txnCommitPeriod) { producer.SendOffsetsToTransaction( // Note: committed offsets reflect the next message to consume, not last // message consumed. consumer.Position returns the last consumed offset // values + 1, as required. consumer.Assignment.Select(a => new TopicPartitionOffset(a, consumer.Position(a))), consumer.ConsumerGroupMetadata, DefaultTimeout); producer.CommitTransaction(); producer.BeginTransaction(); Console.WriteLine($"Committed AggregateWords transaction(s) comprising updates to {wCount} words."); lastTxnCommit = DateTime.Now; wCount = 0; } } catch (Exception e) { producer.AbortTransaction(); Console.WriteLine("Exiting AggregateWords consume loop due to an exception: " + e); consumer.Close(); Console.WriteLine("AggregateWords consumer closed"); break; } } } }
/// <summary> /// A transactional (exactly once) processing loop that reads lines of text from /// Topic_InputLines, splits them into words, and outputs the result to Topic_Words. /// </summary> static void Processor_MapWords(string brokerList, string clientId, CancellationToken ct) { if (clientId == null) { throw new Exception("Map processor requires that a client id is specified."); } var pConfig = new ProducerConfig { BootstrapServers = brokerList, ClientId = clientId + "_producer", // The TransactionalId identifies this instance of the map words processor. // If you start another instance with the same transactional id, the existing // instance will be fenced. TransactionalId = TransactionalIdPrefix_MapWords + "-" + clientId }; var cConfig = new ConsumerConfig { BootstrapServers = brokerList, ClientId = clientId + "_consumer", GroupId = ConsumerGroup_MapWords, // AutoOffsetReset specifies the action to take when there // are no committed offsets for a partition, or an error // occurs retrieving offsets. If there are committed offsets, // it has no effect. AutoOffsetReset = AutoOffsetReset.Earliest, // Offsets are committed using the producer as part of the // transaction - not the consumer. When using transactions, // you must turn off auto commit on the consumer, which is // enabled by default! EnableAutoCommit = false, // Enable incremental rebalancing by using the CooperativeSticky // assignor (avoid stop-the-world rebalances). PartitionAssignmentStrategy = PartitionAssignmentStrategy.CooperativeSticky }; var txnCommitPeriod = TimeSpan.FromSeconds(10); var lastTxnCommit = DateTime.Now; using (var producer = new ProducerBuilder <string, Null>(pConfig).Build()) using (var consumer = new ConsumerBuilder <Null, string>(cConfig) .SetPartitionsRevokedHandler((c, partitions) => { var remaining = c.Assignment.Where(tp => partitions.Where(x => x.TopicPartition == tp).Count() == 0); Console.WriteLine( "** MapWords consumer group partitions revoked: [" + string.Join(',', partitions.Select(p => p.Partition.Value)) + "], remaining: [" + string.Join(',', remaining.Select(p => p.Partition.Value)) + "]"); // All handlers (except the log handler) are executed as a // side-effect of, and on the same thread as the Consume or // Close methods. Any exception thrown in a handler (with // the exception of the log and error handlers) will // be propagated to the application via the initiating // call. i.e. in this example, any exceptions thrown in this // handler will be exposed via the Consume method in the main // consume loop and handled by the try/catch block there. producer.SendOffsetsToTransaction( c.Assignment.Select(a => new TopicPartitionOffset(a, c.Position(a))), c.ConsumerGroupMetadata, DefaultTimeout); producer.CommitTransaction(); producer.BeginTransaction(); }) .SetPartitionsLostHandler((c, partitions) => { // Ownership of the partitions has been involuntarily lost and // are now likely already owned by another consumer. Console.WriteLine( "** MapWords consumer group partitions lost: [" + string.Join(',', partitions.Select(p => p.Partition.Value)) + "]"); producer.AbortTransaction(); producer.BeginTransaction(); }) .SetPartitionsAssignedHandler((c, partitions) => { Console.WriteLine( "** MapWords consumer group additional partitions assigned: [" + string.Join(',', partitions.Select(p => p.Partition.Value)) + "], all: [" + string.Join(',', c.Assignment.Concat(partitions).Select(p => p.Partition.Value)) + "]"); // No action is required here related to transactions - offsets // for the newly assigned partitions will be committed in the // main consume loop along with those for already assigned // partitions as per usual. }) .Build()) { consumer.Subscribe(Topic_InputLines); producer.InitTransactions(DefaultTimeout); producer.BeginTransaction(); var wCount = 0; var lCount = 0; while (true) { try { ct.ThrowIfCancellationRequested(); // Do not block on Consume indefinitely to avoid the possibility of a transaction timeout. var cr = consumer.Consume(TimeSpan.FromSeconds(1)); if (cr != null) { lCount += 1; var words = Regex.Split(cr.Message.Value.ToLower(), @"[^a-zA-Z_]").Where(s => s != String.Empty); foreach (var w in words) { while (true) { try { producer.Produce(Topic_Words, new Message <string, Null> { Key = w }); // Note: when using transactions, there is no need to check for errors of individual // produce call delivery reports because if the transaction commits successfully, you // can be sure that all the constituent messages were delivered successfully and in order. wCount += 1; } catch (KafkaException e) { // An immediate failure of the produce call is most often caused by the // local message queue being full, and appropriate response to that is // to wait a bit and retry. if (e.Error.Code == ErrorCode.Local_QueueFull) { Thread.Sleep(TimeSpan.FromMilliseconds(1000)); continue; } throw; } break; } } } // Commit transactions every TxnCommitPeriod if (DateTime.Now > lastTxnCommit + txnCommitPeriod) { // Note: Exceptions thrown by SendOffsetsToTransaction and // CommitTransaction that are not marked as fatal can be // recovered from. However, in order to keep this example // short(er), the additional logic required to achieve this // has been omitted. This should happen only rarely, so // requiring a process restart in this case is not necessarily // a bad compromise, even in production scenarios. producer.SendOffsetsToTransaction( // Note: committed offsets reflect the next message to consume, not last // message consumed. consumer.Position returns the last consumed offset // values + 1, as required. consumer.Assignment.Select(a => new TopicPartitionOffset(a, consumer.Position(a))), consumer.ConsumerGroupMetadata, DefaultTimeout); producer.CommitTransaction(); producer.BeginTransaction(); Console.WriteLine($"Committed MapWords transaction(s) comprising {wCount} words from {lCount} lines."); lastTxnCommit = DateTime.Now; wCount = 0; lCount = 0; } } catch (Exception e) { // Attempt to abort the transaction (but ignore any errors) as a measure // against stalling consumption of Topic_Words. producer.AbortTransaction(); Console.WriteLine("Exiting MapWords consume loop due to an exception: " + e); // Note: transactions may be committed / aborted in the partitions // revoked / lost handler as a side effect of the call to close. consumer.Close(); Console.WriteLine("MapWords consumer closed"); break; } // Assume the presence of an external system that monitors whether worker // processes have died, and restarts new instances as required. This // setup is typical, and avoids complex error handling logic in the // client code. } } }