public void Run(string groupId, bool returnOnLastOffset)
{
var consumerConfig = new ConsumerConfig
{
GroupId = groupId,
BootstrapServers = "localhost:9092",
AutoOffsetReset = AutoOffsetReset.Earliest,
};
using (var c = new ConsumerBuilder <int, string>(consumerConfig).Build())
{
c.Subscribe("products.cache");
try
{
var watermark = c.QueryWatermarkOffsets(new TopicPartition("products.cache", new Partition(0)), TimeSpan.FromMilliseconds(10000));
if (returnOnLastOffset && watermark.High.Value == 0)
{
return;
}
while (true)
{
try
{
ConsumeResult <int, string> cr = c.Consume();
if (cr.Value == null)
{
if (_cache.TryGetValue(cr.Key, out _))
{
_cache.Remove(cr.Key);
}
}
else
{
var item = JsonConvert.DeserializeObject <ProductCacheItem>(cr.Value);
_cache.Set(cr.Key, item);
}
if (returnOnLastOffset && watermark.High.Value - 1 == cr.Offset.Value)
{
return;
}
}
catch (Exception e)
{
//Log e
throw;
}
}
}
catch (Exception ex)
{
//Log ex
throw;
}
}
}
public void WatermarkOffsets(string bootstrapServers)
{
LogToFile("start WatermarkOffsets");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var testString = "hello world";
DeliveryResult <Null, string> dr;
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
{
dr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> {
Value = testString
}).Result;
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); // this isn't necessary.
}
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000
};
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new List <TopicPartitionOffset>()
{
dr.TopicPartitionOffset
});
var record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
var getOffsets = consumer.GetWatermarkOffsets(dr.TopicPartition);
Assert.Equal(getOffsets.Low, Offset.Unset);
// the offset of the next message to be read.
Assert.Equal(getOffsets.High, dr.Offset + 1);
var queryOffsets = consumer.QueryWatermarkOffsets(dr.TopicPartition, TimeSpan.FromSeconds(20));
Assert.NotEqual(queryOffsets.Low, Offset.Unset);
Assert.Equal(getOffsets.High, queryOffsets.High);
}
// Test empty topic case
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
var wo = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(30));
// Refer to WatermarkOffsets class documentation for more information.
Assert.Equal(0, wo.Low);
Assert.Equal(0, wo.High);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end WatermarkOffsets");
}
/// <summary>
/// Extract messages using filter after specified time
/// </summary>
/// <param name="consumerConfig"></param>
/// <param name="topic"></param>
/// <param name="partition"></param>
/// <param name="offset"></param>
/// <param name="filter"></param>
/// <param name="timeAfter"></param>
static void AssignManually(ConsumerConfig consumerConfig, string topic, int partition, long offset, string filter, DateTime timeAfter)
{
string message = null;
long ExtractedCountWithFilter = 0;
using (var consumer = new ConsumerBuilder <Ignore, string>(consumerConfig).SetErrorHandler((_, e) => logger.LogError($"Error: {e.Reason}")).Build())
{
// Get last offset of specified partition
long lastoffset = consumer.QueryWatermarkOffsets(new TopicPartition(topic, new Partition(partition)), TimeSpan.FromSeconds(10)).High - 1;
// Extract messages from offset
consumer.Assign(new TopicPartitionOffset(topic, partition, offset));
try
{
while (true)
{
try
{
var consumeResult = consumer.Consume(TimeSpan.FromSeconds(10));
// Note: End of partition notification has not been enabled, so
// it is guaranteed that the ConsumeResult instance corresponds
// to a Message, and not a PartitionEOF event.
if (consumeResult.Message.Timestamp.UtcDateTime.CompareTo(timeAfter) < 0)
{
logger.LogInformation($"Skip message at {consumeResult.TopicPartitionOffset} as it is ealier than {timeAfter}");
continue;
}
message = consumeResult.Message.Value;
logger.LogInformation($"Received message at {consumeResult.TopicPartitionOffset}");
if (string.IsNullOrWhiteSpace(filter) || message.Contains(filter)) // if filter is actually empty or message contains filter, extract the message
{
MessageList.Enqueue(message);
ExtractedCountWithFilter++;
MessageExtractedCount++;
logger.LogInformation("Message extracted");
}
if (consumeResult.TopicPartitionOffset.Offset == lastoffset)
{
break;
}
}
catch (ConsumeException e)
{
logger.LogError($"Consume error: {e.Error.Reason}");
}
}
}
catch (OperationCanceledException)
{
logger.LogInformation("Closing consumer.");
consumer.Close();
}
}
logger.LogInformation($"{ExtractedCountWithFilter} extracted from partition {partition}");
}
public long GetFlowSize(IMessageFlow messageFlow)
{
var settings = _kafkaSettingsFactory.CreateReceiverSettings(messageFlow);
var topicPartition = new TopicPartition(settings.TopicPartitionOffset.Topic, ZeroPartition);
using var consumer = new ConsumerBuilder <Ignore, Ignore>(settings.Config).Build();
var offsets = consumer.QueryWatermarkOffsets(topicPartition, settings.PollTimeout);
return(offsets.High);
}
public ConsumeResult <Ignore, Ignore> TryGetFlowLastMessage(IMessageFlow messageFlow)
{
var settings = _kafkaSettingsFactory.CreateReceiverSettings(messageFlow);
var topicPartition = new TopicPartition(settings.TopicPartitionOffset.Topic, ZeroPartition);
using var consumer = new ConsumerBuilder <Ignore, Ignore>(settings.Config).Build();
var offsets = consumer.QueryWatermarkOffsets(topicPartition, settings.PollTimeout);
consumer.Assign(new[] { new TopicPartitionOffset(topicPartition, offsets.High - 1) });
return(consumer.Consume(settings.PollTimeout));
}
public static void WatermarkOffsets(string bootstrapServers, string singlePartitionTopic, string partitionedTopic)
{
LogToFile("start WatermarkOffsets");
var producerConfig = new ProducerConfig {
BootstrapServers = bootstrapServers
};
var testString = "hello world";
DeliveryResult <Null, string> dr;
using (var producer = new ProducerBuilder <Null, string>(producerConfig).Build())
using (var adminClient = new AdminClient(producer.Handle))
{
dr = producer.ProduceAsync(singlePartitionTopic, new Message <Null, string> {
Value = testString
}).Result;
Assert.Equal(0, producer.Flush(TimeSpan.FromSeconds(10))); // this isn't necessary.
}
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000
};
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new List <TopicPartitionOffset>()
{
dr.TopicPartitionOffset
});
var record = consumer.Consume(TimeSpan.FromSeconds(10));
Assert.NotNull(record.Message);
var getOffsets = consumer.GetWatermarkOffsets(dr.TopicPartition);
Assert.Equal(getOffsets.Low, Offset.Invalid);
// the offset of the next message to be read.
Assert.Equal(getOffsets.High, dr.Offset + 1);
var queryOffsets = consumer.QueryWatermarkOffsets(dr.TopicPartition, TimeSpan.FromSeconds(20));
Assert.NotEqual(queryOffsets.Low, Offset.Invalid);
Assert.Equal(getOffsets.High, queryOffsets.High);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end WatermarkOffsets");
}
public IReadOnlyCollection <MessageFlowStats> GetFlowStats(IMessageFlow messageFlow)
{
var settings = _kafkaSettingsFactory.CreateReceiverSettings(messageFlow);
using var consumer = new ConsumerBuilder <Ignore, Ignore>(settings.Config).Build();
var topicPartitions = GetTopicPartitions(settings.TopicPartitionOffsets.Select(x => x.Topic));
var stats = consumer.Committed(topicPartitions, settings.PollTimeout).Select(x =>
{
var offsets = consumer.QueryWatermarkOffsets(x.TopicPartition, settings.PollTimeout);
return(new MessageFlowStats(x.TopicPartition, offsets.High, x.Offset));
}).ToList();
return(stats);
}
public void Consumer_Drain(string bootstrapServers)
{
LogToFile("start Consumer_Drain");
int N = 142;
var firstProduced = Util.ProduceNullStringMessages(bootstrapServers, singlePartitionTopic, 100, N);
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
EnableAutoCommit = false,
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
Util.ProduceNullStringMessages(bootstrapServers, topic.Name, 100, N);
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
var offsets = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(10));
Assert.Equal(0, offsets.Low);
Assert.Equal(N, offsets.High); // offsets.High is the next message to be read == the offset of last message + 1.
consumer.Commit(new[] { new TopicPartitionOffset(topic.Name, 0, new Offset(offsets.High)) });
consumer.Subscribe(topic.Name);
var cnt = 0;
while (consumer.Assignment.Count == 0)
{
Thread.Sleep(1000);
Assert.True(cnt++ < 10);
}
var committed = consumer.Committed(TimeSpan.FromSeconds(10));
Assert.Single(committed);
Assert.Equal(N, committed[0].Offset);
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Consumer_Drain");
}
public override (ISenderQueue <Item>, IBlockingReceiverQueue <Item>) Create()
{
ClientConfig clientConfig;
var topic = "items";
var localKafka = true;
// Local Kafka
if (localKafka)
{
var bootstrapServers = "localhost:9092";
clientConfig = new ClientConfig
{
BootstrapServers = bootstrapServers
};
}
//Azure Event Hub
else
{
var fqdn = "";
var connectionString = "";
clientConfig = new ClientConfig
{
BootstrapServers = fqdn,
SecurityProtocol = SecurityProtocol.SaslSsl,
SaslMechanism = SaslMechanism.Plain,
SaslUsername = "******",
SaslPassword = connectionString,
};
}
var consumerConfig = new ConsumerConfig(clientConfig)
{
GroupId = "default"
};
var adminClient = new AdminClientBuilder(clientConfig).Build();
var metadata = adminClient.GetMetadata(TimeSpan.FromSeconds(5));
var topicMetadata = metadata.Topics.FirstOrDefault(t => t.Topic == topic);
if (topicMetadata != null)
{
var consumer = new ConsumerBuilder <Ignore, Item>(consumerConfig)
.SetValueDeserializer(new JsonDeserializer <Item>())
.Build();
foreach (var partition in topicMetadata.Partitions)
{
var topicPartition = new TopicPartition(topic, new Partition(partition.PartitionId));
var offSet = consumer.QueryWatermarkOffsets(topicPartition, TimeSpan.FromSeconds(5));
consumer.Commit(new TopicPartitionOffset[] { new TopicPartitionOffset(topicPartition, offSet.High) });
}
consumer.Close();
}
var senderQueue = new KafkaSenderQueue <Item>(new ProducerConfig(clientConfig), topic, new JsonItemSerializer());
var receiverQueue = new KafkaReceiverQueue <Item>(consumerConfig, topic, new JsonItemSerializer());
receiverQueue.OpenAsync(CancellationToken).Wait();
return(senderQueue, receiverQueue);
}
public async Task <IEnumerable <Models.KafkaMessageModel> > GetMessages(string topic, long count = -1)
{
List <Models.KafkaMessageModel> toReturn = new List <Models.KafkaMessageModel>();
if (!string.IsNullOrEmpty(this.BootstrapServers) && !string.IsNullOrEmpty(this.GroupId))
{
ConsumerConfig config = new ConsumerConfig()
{
BootstrapServers = this.BootstrapServers,
GroupId = this.GroupId,
AutoOffsetReset = AutoOffsetReset.Latest
};
await Task.Run(() =>
{
using (IConsumer <string, string> consumer = new ConsumerBuilder <string, string>(config).Build())
{
ConsumeResult <string, string> result = null;
try
{
consumer.Subscribe(topic);
while (!consumer.Assignment.Any())
{
}
TopicPartition tp = consumer.Assignment.FirstOrDefault();
WatermarkOffsets wo = consumer.QueryWatermarkOffsets(tp, TimeSpan.FromSeconds(this.TimeoutSeconds));
long numMessages = wo.High - wo.Low;
if (count > 0 && count < numMessages)
{
numMessages = count;
}
consumer.Seek(new TopicPartitionOffset(tp, wo.High - numMessages));
do
{
result = consumer.Consume(TimeSpan.FromSeconds(this.TimeoutSeconds));
if (result != null)
{
try
{
toReturn.Add(new Models.KafkaMessageModel()
{
Topic = result.Topic,
Value = JsonConvert.DeserializeObject <MessageModel>(result.Message.Value),
Raw = result.Message.Value
});
}
catch (JsonSerializationException) { } /* We may add events in the future, and don't want to stop collecting current events if we haven't accounted for the structure */
}
} while (result != null && result.TopicPartitionOffset.Offset.Value <= wo.High - 1);
}
catch (Exception) { }
consumer.Unsubscribe();
consumer.Close();
}
});
}
return(toReturn);
}
public void Transactions_WatermarkOffsets(string bootstrapServers)
{
LogToFile("start Transactions_WatermarkOffsets");
var groupName = Guid.NewGuid().ToString();
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString(), LingerMs = 0
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
IsolationLevel = IsolationLevel.ReadCommitted, BootstrapServers = bootstrapServers, GroupId = groupName, EnableAutoCommit = false
}).Build())
{
var wo1 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
Assert.Equal(Offset.Unset, wo1.Low);
Assert.Equal(Offset.Unset, wo1.High);
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
producer.InitTransactions(TimeSpan.FromSeconds(30));
producer.BeginTransaction();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message1"
}).Wait();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message2"
}).Wait();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message3"
}).Wait();
WatermarkOffsets wo2 = new WatermarkOffsets(Offset.Unset, Offset.Unset);
for (int i = 0; i < 10; ++i)
{
var cr = consumer.Consume(TimeSpan.FromMilliseconds(500));
wo2 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
if (wo2.High == 3)
{
break;
}
}
Assert.Equal(3, wo2.High);
producer.CommitTransaction(TimeSpan.FromSeconds(30));
WatermarkOffsets wo3 = new WatermarkOffsets(Offset.Unset, Offset.Unset);
for (int i = 0; i < 10; ++i)
{
var cr2 = consumer.Consume(TimeSpan.FromSeconds(500));
wo3 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
if (wo3.High > 3)
{
break;
}
}
Assert.Equal(4, wo3.High);
var wo4 = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(30));
Assert.Equal(4, wo4.High);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_WatermarkOffsets");
}
public void Transactions_Commit(string bootstrapServers)
{
LogToFile("start Transactions_Commit");
var defaultTimeout = TimeSpan.FromSeconds(30);
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString()
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = "unimportant", EnableAutoCommit = false, Debug = "all"
}).Build())
{
var wm = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), defaultTimeout);
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, wm.High));
producer.InitTransactions(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 0", Value = "test val 0"
});
producer.CommitTransaction(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 1", Value = "test val 1"
});
producer.CommitTransaction(defaultTimeout);
var cr1 = consumer.Consume();
var cr2 = consumer.Consume();
var cr3 = consumer.Consume(TimeSpan.FromMilliseconds(100)); // force the consumer to read over the final control message internally.
Assert.Equal(wm.High, cr1.Offset);
Assert.Equal(wm.High + 2, cr2.Offset); // there should be a skipped offset due to a commit marker in the log.
Assert.Null(cr3); // control message should not be exposed to application.
// Test that the committed offset accounts for the final ctrl message.
consumer.Commit();
}
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString()
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = "unimportant", EnableAutoCommit = false, AutoOffsetReset = AutoOffsetReset.Latest
}).Build())
{
consumer.Assign(new TopicPartition(topic.Name, 0));
// call InitTransactions to prevent a race conidtion between a slow txn commit and a quick offset request.
producer.InitTransactions(defaultTimeout);
var committed = consumer.Committed(defaultTimeout);
var wm = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), defaultTimeout);
Assert.Equal(wm.High, committed[0].Offset);
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_Commit");
}
public void AdminClient_DeleteRecords(string bootstrapServers)
{
LogToFile("start AdminClient_DeleteRecords");
using (var topic1 = new TemporaryTopic(bootstrapServers, 1))
using (var topic2 = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <Null, string>(new ProducerConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var adminClient = new AdminClientBuilder(new AdminClientConfig {
BootstrapServers = bootstrapServers
}).Build())
using (var consumer = new ConsumerBuilder <Null, string>(new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = "unimportant"
}).Build())
{
for (int i = 0; i < 10; ++i)
{
producer.Produce(topic1.Name, new Message <Null, string> {
Value = i.ToString()
});
producer.Produce(topic2.Name, new Message <Null, string> {
Value = i.ToString()
});
}
producer.Flush(TimeSpan.FromSeconds(10));
var r = adminClient.DeleteRecordsAsync(new List <TopicPartitionOffset>
{
new TopicPartitionOffset(topic1.Name, 0, 4),
new TopicPartitionOffset(topic2.Name, 0, 0) // no modification.
}).Result;
var t1r = r.Where(a => a.Topic == topic1.Name).ToList()[0];
var t2r = r.Where(a => a.Topic == topic2.Name).ToList()[0];
Assert.Equal(4, (int)t1r.Offset);
Assert.Equal(0, (int)t2r.Offset);
var wm1 = consumer.QueryWatermarkOffsets(new TopicPartition(topic1.Name, 0), TimeSpan.FromSeconds(10));
Assert.Equal(4, (int)wm1.Low);
var wm2 = consumer.QueryWatermarkOffsets(new TopicPartition(topic2.Name, 0), TimeSpan.FromSeconds(10));
Assert.Equal(0, (int)wm2.Low);
r = adminClient.DeleteRecordsAsync(new List <TopicPartitionOffset>
{
new TopicPartitionOffset(topic1.Name, 0, 3)
}).Result;
Assert.Equal(4, (int)r.First().Offset);
try
{
r = adminClient.DeleteRecordsAsync(new List <TopicPartitionOffset>
{
new TopicPartitionOffset(topic1.Name, 0, 12)
}).Result;
Assert.True(false); // expecting exception.
}
catch (Exception e)
{
var ie = e.InnerException;
Assert.IsType <Admin.DeleteRecordsException>(ie);
var dre = (Admin.DeleteRecordsException)ie;
Assert.Single(dre.Results);
Assert.Equal(ErrorCode.OffsetOutOfRange, dre.Results[0].Error.Code);
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end AdminClient_DeleteRecords");
}