public void Run(string groupId, bool returnOnLastOffset) { var consumerConfig = new ConsumerConfig { GroupId = groupId, BootstrapServers = "localhost:9092", AutoOffsetReset = AutoOffsetReset.Earliest, }; using (var c = new ConsumerBuilder <int, string>(consumerConfig).Build()) { c.Subscribe("products.cache"); try { var watermark = c.QueryWatermarkOffsets(new TopicPartition("products.cache", new Partition(0)), TimeSpan.FromMilliseconds(10000)); if (returnOnLastOffset && watermark.High.Value == 0) { return; } while (true) { try { ConsumeResult <int, string> cr = c.Consume(); if (cr.Value == null) { if (_cache.TryGetValue(cr.Key, out _)) { _cache.Remove(cr.Key); } } else { var item = JsonConvert.DeserializeObject <ProductCacheItem>(cr.Value); _cache.Set(cr.Key, item); } if (returnOnLastOffset && watermark.High.Value - 1 == cr.Offset.Value) { return; } } catch (Exception e) { //Log e throw; } } } catch (Exception ex) { //Log ex throw; } } }
public void WatermarkOffsets(string bootstrapServers) { LogToFile("start WatermarkOffsets"); var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var testString = "hello world"; DeliveryResult <Null, string> dr; using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build()) { dr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> { Value = testString }).Result; Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); // this isn't necessary. } var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, SessionTimeoutMs = 6000 }; using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new List <TopicPartitionOffset>() { dr.TopicPartitionOffset }); var record = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.NotNull(record.Message); var getOffsets = consumer.GetWatermarkOffsets(dr.TopicPartition); Assert.Equal(getOffsets.Low, Offset.Unset); // the offset of the next message to be read. Assert.Equal(getOffsets.High, dr.Offset + 1); var queryOffsets = consumer.QueryWatermarkOffsets(dr.TopicPartition, TimeSpan.FromSeconds(20)); Assert.NotEqual(queryOffsets.Low, Offset.Unset); Assert.Equal(getOffsets.High, queryOffsets.High); } // Test empty topic case using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { var wo = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(30)); // Refer to WatermarkOffsets class documentation for more information. Assert.Equal(0, wo.Low); Assert.Equal(0, wo.High); } Assert.Equal(0, Library.HandleCount); LogToFile("end WatermarkOffsets"); }
/// <summary> /// Extract messages using filter after specified time /// </summary> /// <param name="consumerConfig"></param> /// <param name="topic"></param> /// <param name="partition"></param> /// <param name="offset"></param> /// <param name="filter"></param> /// <param name="timeAfter"></param> static void AssignManually(ConsumerConfig consumerConfig, string topic, int partition, long offset, string filter, DateTime timeAfter) { string message = null; long ExtractedCountWithFilter = 0; using (var consumer = new ConsumerBuilder <Ignore, string>(consumerConfig).SetErrorHandler((_, e) => logger.LogError($"Error: {e.Reason}")).Build()) { // Get last offset of specified partition long lastoffset = consumer.QueryWatermarkOffsets(new TopicPartition(topic, new Partition(partition)), TimeSpan.FromSeconds(10)).High - 1; // Extract messages from offset consumer.Assign(new TopicPartitionOffset(topic, partition, offset)); try { while (true) { try { var consumeResult = consumer.Consume(TimeSpan.FromSeconds(10)); // Note: End of partition notification has not been enabled, so // it is guaranteed that the ConsumeResult instance corresponds // to a Message, and not a PartitionEOF event. if (consumeResult.Message.Timestamp.UtcDateTime.CompareTo(timeAfter) < 0) { logger.LogInformation($"Skip message at {consumeResult.TopicPartitionOffset} as it is ealier than {timeAfter}"); continue; } message = consumeResult.Message.Value; logger.LogInformation($"Received message at {consumeResult.TopicPartitionOffset}"); if (string.IsNullOrWhiteSpace(filter) || message.Contains(filter)) // if filter is actually empty or message contains filter, extract the message { MessageList.Enqueue(message); ExtractedCountWithFilter++; MessageExtractedCount++; logger.LogInformation("Message extracted"); } if (consumeResult.TopicPartitionOffset.Offset == lastoffset) { break; } } catch (ConsumeException e) { logger.LogError($"Consume error: {e.Error.Reason}"); } } } catch (OperationCanceledException) { logger.LogInformation("Closing consumer."); consumer.Close(); } } logger.LogInformation($"{ExtractedCountWithFilter} extracted from partition {partition}"); }
public long GetFlowSize(IMessageFlow messageFlow) { var settings = _kafkaSettingsFactory.CreateReceiverSettings(messageFlow); var topicPartition = new TopicPartition(settings.TopicPartitionOffset.Topic, ZeroPartition); using var consumer = new ConsumerBuilder <Ignore, Ignore>(settings.Config).Build(); var offsets = consumer.QueryWatermarkOffsets(topicPartition, settings.PollTimeout); return(offsets.High); }
public ConsumeResult <Ignore, Ignore> TryGetFlowLastMessage(IMessageFlow messageFlow) { var settings = _kafkaSettingsFactory.CreateReceiverSettings(messageFlow); var topicPartition = new TopicPartition(settings.TopicPartitionOffset.Topic, ZeroPartition); using var consumer = new ConsumerBuilder <Ignore, Ignore>(settings.Config).Build(); var offsets = consumer.QueryWatermarkOffsets(topicPartition, settings.PollTimeout); consumer.Assign(new[] { new TopicPartitionOffset(topicPartition, offsets.High - 1) }); return(consumer.Consume(settings.PollTimeout)); }
public static void WatermarkOffsets(string bootstrapServers, string singlePartitionTopic, string partitionedTopic) { LogToFile("start WatermarkOffsets"); var producerConfig = new ProducerConfig { BootstrapServers = bootstrapServers }; var testString = "hello world"; DeliveryResult <Null, string> dr; using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build()) using (var adminClient = new AdminClient(producer.Handle)) { dr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> { Value = testString }).Result; Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); // this isn't necessary. } var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, SessionTimeoutMs = 6000 }; using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { consumer.Assign(new List <TopicPartitionOffset>() { dr.TopicPartitionOffset }); var record = consumer.Consume(TimeSpan.FromSeconds(10)); Assert.NotNull(record.Message); var getOffsets = consumer.GetWatermarkOffsets(dr.TopicPartition); Assert.Equal(getOffsets.Low, Offset.Invalid); // the offset of the next message to be read. Assert.Equal(getOffsets.High, dr.Offset + 1); var queryOffsets = consumer.QueryWatermarkOffsets(dr.TopicPartition, TimeSpan.FromSeconds(20)); Assert.NotEqual(queryOffsets.Low, Offset.Invalid); Assert.Equal(getOffsets.High, queryOffsets.High); } Assert.Equal(0, Library.HandleCount); LogToFile("end WatermarkOffsets"); }
public IReadOnlyCollection <MessageFlowStats> GetFlowStats(IMessageFlow messageFlow) { var settings = _kafkaSettingsFactory.CreateReceiverSettings(messageFlow); using var consumer = new ConsumerBuilder <Ignore, Ignore>(settings.Config).Build(); var topicPartitions = GetTopicPartitions(settings.TopicPartitionOffsets.Select(x => x.Topic)); var stats = consumer.Committed(topicPartitions, settings.PollTimeout).Select(x => { var offsets = consumer.QueryWatermarkOffsets(x.TopicPartition, settings.PollTimeout); return(new MessageFlowStats(x.TopicPartition, offsets.High, x.Offset)); }).ToList(); return(stats); }
public void Consumer_Drain(string bootstrapServers) { LogToFile("start Consumer_Drain"); int N = 142; var firstProduced = Util.ProduceNullStringMessages(bootstrapServers, singlePartitionTopic, 100, N); var consumerConfig = new ConsumerConfig { GroupId = Guid.NewGuid().ToString(), BootstrapServers = bootstrapServers, EnableAutoCommit = false, }; using (var topic = new TemporaryTopic(bootstrapServers, 1)) { Util.ProduceNullStringMessages(bootstrapServers, topic.Name, 100, N); using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build()) { var offsets = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(10)); Assert.Equal(0, offsets.Low); Assert.Equal(N, offsets.High); // offsets.High is the next message to be read == the offset of last message + 1. consumer.Commit(new[] { new TopicPartitionOffset(topic.Name, 0, new Offset(offsets.High)) }); consumer.Subscribe(topic.Name); var cnt = 0; while (consumer.Assignment.Count == 0) { Thread.Sleep(1000); Assert.True(cnt++ < 10); } var committed = consumer.Committed(TimeSpan.FromSeconds(10)); Assert.Single(committed); Assert.Equal(N, committed[0].Offset); } } Assert.Equal(0, Library.HandleCount); LogToFile("end Consumer_Drain"); }
public override (ISenderQueue <Item>, IBlockingReceiverQueue <Item>) Create() { ClientConfig clientConfig; var topic = "items"; var localKafka = true; // Local Kafka if (localKafka) { var bootstrapServers = "localhost:9092"; clientConfig = new ClientConfig { BootstrapServers = bootstrapServers }; } //Azure Event Hub else { var fqdn = ""; var connectionString = ""; clientConfig = new ClientConfig { BootstrapServers = fqdn, SecurityProtocol = SecurityProtocol.SaslSsl, SaslMechanism = SaslMechanism.Plain, SaslUsername = "******", SaslPassword = connectionString, }; } var consumerConfig = new ConsumerConfig(clientConfig) { GroupId = "default" }; var adminClient = new AdminClientBuilder(clientConfig).Build(); var metadata = adminClient.GetMetadata(TimeSpan.FromSeconds(5)); var topicMetadata = metadata.Topics.FirstOrDefault(t => t.Topic == topic); if (topicMetadata != null) { var consumer = new ConsumerBuilder <Ignore, Item>(consumerConfig) .SetValueDeserializer(new JsonDeserializer <Item>()) .Build(); foreach (var partition in topicMetadata.Partitions) { var topicPartition = new TopicPartition(topic, new Partition(partition.PartitionId)); var offSet = consumer.QueryWatermarkOffsets(topicPartition, TimeSpan.FromSeconds(5)); consumer.Commit(new TopicPartitionOffset[] { new TopicPartitionOffset(topicPartition, offSet.High) }); } consumer.Close(); } var senderQueue = new KafkaSenderQueue <Item>(new ProducerConfig(clientConfig), topic, new JsonItemSerializer()); var receiverQueue = new KafkaReceiverQueue <Item>(consumerConfig, topic, new JsonItemSerializer()); receiverQueue.OpenAsync(CancellationToken).Wait(); return(senderQueue, receiverQueue); }
public async Task <IEnumerable <Models.KafkaMessageModel> > GetMessages(string topic, long count = -1) { List <Models.KafkaMessageModel> toReturn = new List <Models.KafkaMessageModel>(); if (!string.IsNullOrEmpty(this.BootstrapServers) && !string.IsNullOrEmpty(this.GroupId)) { ConsumerConfig config = new ConsumerConfig() { BootstrapServers = this.BootstrapServers, GroupId = this.GroupId, AutoOffsetReset = AutoOffsetReset.Latest }; await Task.Run(() => { using (IConsumer <string, string> consumer = new ConsumerBuilder <string, string>(config).Build()) { ConsumeResult <string, string> result = null; try { consumer.Subscribe(topic); while (!consumer.Assignment.Any()) { } TopicPartition tp = consumer.Assignment.FirstOrDefault(); WatermarkOffsets wo = consumer.QueryWatermarkOffsets(tp, TimeSpan.FromSeconds(this.TimeoutSeconds)); long numMessages = wo.High - wo.Low; if (count > 0 && count < numMessages) { numMessages = count; } consumer.Seek(new TopicPartitionOffset(tp, wo.High - numMessages)); do { result = consumer.Consume(TimeSpan.FromSeconds(this.TimeoutSeconds)); if (result != null) { try { toReturn.Add(new Models.KafkaMessageModel() { Topic = result.Topic, Value = JsonConvert.DeserializeObject <MessageModel>(result.Message.Value), Raw = result.Message.Value }); } catch (JsonSerializationException) { } /* We may add events in the future, and don't want to stop collecting current events if we haven't accounted for the structure */ } } while (result != null && result.TopicPartitionOffset.Offset.Value <= wo.High - 1); } catch (Exception) { } consumer.Unsubscribe(); consumer.Close(); } }); } return(toReturn); }
public void Transactions_WatermarkOffsets(string bootstrapServers) { LogToFile("start Transactions_WatermarkOffsets"); var groupName = Guid.NewGuid().ToString(); using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var producer = new ProducerBuilder <string, string>(new ProducerConfig { BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString(), LingerMs = 0 }).Build()) using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig { IsolationLevel = IsolationLevel.ReadCommitted, BootstrapServers = bootstrapServers, GroupId = groupName, EnableAutoCommit = false }).Build()) { var wo1 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0)); Assert.Equal(Offset.Unset, wo1.Low); Assert.Equal(Offset.Unset, wo1.High); consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0)); producer.InitTransactions(TimeSpan.FromSeconds(30)); producer.BeginTransaction(); producer.ProduceAsync(topic.Name, new Message <string, string> { Key = "test", Value = "message1" }).Wait(); producer.ProduceAsync(topic.Name, new Message <string, string> { Key = "test", Value = "message2" }).Wait(); producer.ProduceAsync(topic.Name, new Message <string, string> { Key = "test", Value = "message3" }).Wait(); WatermarkOffsets wo2 = new WatermarkOffsets(Offset.Unset, Offset.Unset); for (int i = 0; i < 10; ++i) { var cr = consumer.Consume(TimeSpan.FromMilliseconds(500)); wo2 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0)); if (wo2.High == 3) { break; } } Assert.Equal(3, wo2.High); producer.CommitTransaction(TimeSpan.FromSeconds(30)); WatermarkOffsets wo3 = new WatermarkOffsets(Offset.Unset, Offset.Unset); for (int i = 0; i < 10; ++i) { var cr2 = consumer.Consume(TimeSpan.FromSeconds(500)); wo3 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0)); if (wo3.High > 3) { break; } } Assert.Equal(4, wo3.High); var wo4 = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(30)); Assert.Equal(4, wo4.High); } Assert.Equal(0, Library.HandleCount); LogToFile("end Transactions_WatermarkOffsets"); }
public void Transactions_Commit(string bootstrapServers) { LogToFile("start Transactions_Commit"); var defaultTimeout = TimeSpan.FromSeconds(30); using (var topic = new TemporaryTopic(bootstrapServers, 1)) { using (var producer = new ProducerBuilder <string, string>(new ProducerConfig { BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString() }).Build()) using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = "unimportant", EnableAutoCommit = false, Debug = "all" }).Build()) { var wm = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), defaultTimeout); consumer.Assign(new TopicPartitionOffset(topic.Name, 0, wm.High)); producer.InitTransactions(defaultTimeout); producer.BeginTransaction(); producer.Produce(topic.Name, new Message <string, string> { Key = "test key 0", Value = "test val 0" }); producer.CommitTransaction(defaultTimeout); producer.BeginTransaction(); producer.Produce(topic.Name, new Message <string, string> { Key = "test key 1", Value = "test val 1" }); producer.CommitTransaction(defaultTimeout); var cr1 = consumer.Consume(); var cr2 = consumer.Consume(); var cr3 = consumer.Consume(TimeSpan.FromMilliseconds(100)); // force the consumer to read over the final control message internally. Assert.Equal(wm.High, cr1.Offset); Assert.Equal(wm.High + 2, cr2.Offset); // there should be a skipped offset due to a commit marker in the log. Assert.Null(cr3); // control message should not be exposed to application. // Test that the committed offset accounts for the final ctrl message. consumer.Commit(); } using (var producer = new ProducerBuilder <string, string>(new ProducerConfig { BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString() }).Build()) using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = "unimportant", EnableAutoCommit = false, AutoOffsetReset = AutoOffsetReset.Latest }).Build()) { consumer.Assign(new TopicPartition(topic.Name, 0)); // call InitTransactions to prevent a race conidtion between a slow txn commit and a quick offset request. producer.InitTransactions(defaultTimeout); var committed = consumer.Committed(defaultTimeout); var wm = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), defaultTimeout); Assert.Equal(wm.High, committed[0].Offset); } } Assert.Equal(0, Library.HandleCount); LogToFile("end Transactions_Commit"); }
public void AdminClient_DeleteRecords(string bootstrapServers) { LogToFile("start AdminClient_DeleteRecords"); using (var topic1 = new TemporaryTopic(bootstrapServers, 1)) using (var topic2 = new TemporaryTopic(bootstrapServers, 1)) using (var producer = new ProducerBuilder <Null, string>(new ProducerConfig { BootstrapServers = bootstrapServers }).Build()) using (var adminClient = new AdminClientBuilder(new AdminClientConfig { BootstrapServers = bootstrapServers }).Build()) using (var consumer = new ConsumerBuilder <Null, string>(new ConsumerConfig { BootstrapServers = bootstrapServers, GroupId = "unimportant" }).Build()) { for (int i = 0; i < 10; ++i) { producer.Produce(topic1.Name, new Message <Null, string> { Value = i.ToString() }); producer.Produce(topic2.Name, new Message <Null, string> { Value = i.ToString() }); } producer.Flush(TimeSpan.FromSeconds(10)); var r = adminClient.DeleteRecordsAsync(new List <TopicPartitionOffset> { new TopicPartitionOffset(topic1.Name, 0, 4), new TopicPartitionOffset(topic2.Name, 0, 0) // no modification. }).Result; var t1r = r.Where(a => a.Topic == topic1.Name).ToList()[0]; var t2r = r.Where(a => a.Topic == topic2.Name).ToList()[0]; Assert.Equal(4, (int)t1r.Offset); Assert.Equal(0, (int)t2r.Offset); var wm1 = consumer.QueryWatermarkOffsets(new TopicPartition(topic1.Name, 0), TimeSpan.FromSeconds(10)); Assert.Equal(4, (int)wm1.Low); var wm2 = consumer.QueryWatermarkOffsets(new TopicPartition(topic2.Name, 0), TimeSpan.FromSeconds(10)); Assert.Equal(0, (int)wm2.Low); r = adminClient.DeleteRecordsAsync(new List <TopicPartitionOffset> { new TopicPartitionOffset(topic1.Name, 0, 3) }).Result; Assert.Equal(4, (int)r.First().Offset); try { r = adminClient.DeleteRecordsAsync(new List <TopicPartitionOffset> { new TopicPartitionOffset(topic1.Name, 0, 12) }).Result; Assert.True(false); // expecting exception. } catch (Exception e) { var ie = e.InnerException; Assert.IsType <Admin.DeleteRecordsException>(ie); var dre = (Admin.DeleteRecordsException)ie; Assert.Single(dre.Results); Assert.Equal(ErrorCode.OffsetOutOfRange, dre.Results[0].Error.Code); } } Assert.Equal(0, Library.HandleCount); LogToFile("end AdminClient_DeleteRecords"); }