public void Transactions_SendOffsets(string bootstrapServers)
{
LogToFile("start Transactions_SendOffsets");
var defaultTimeout = TimeSpan.FromSeconds(30);
var groupName = Guid.NewGuid().ToString();
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString()
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
IsolationLevel = IsolationLevel.ReadCommitted, BootstrapServers = bootstrapServers, GroupId = groupName, EnableAutoCommit = false, Debug = "all"
}).Build())
{
producer.InitTransactions(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 0", Value = "test val 0"
});
producer.SendOffsetsToTransaction(new List <TopicPartitionOffset> {
new TopicPartitionOffset(topic.Name, 0, 7324)
}, consumer.ConsumerGroupMetadata, TimeSpan.FromSeconds(30));
producer.CommitTransaction(defaultTimeout);
var committed = consumer.Committed(new List <TopicPartition> {
new TopicPartition(topic.Name, 0)
}, TimeSpan.FromSeconds(30));
Assert.Single(committed);
Assert.Equal(7324, committed[0].Offset);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_SendOffsets");
}
public long GetFlowProcessedSize(IMessageFlow messageFlow)
{
var settings = _kafkaSettingsFactory.CreateReceiverSettings(messageFlow);
var topicPartition = new TopicPartition(settings.TopicPartitionOffset.Topic, ZeroPartition);
using var consumer = new ConsumerBuilder <Ignore, Ignore>(settings.Config).Build();
var committedOffset = consumer.Committed(new[] { topicPartition }, settings.PollTimeout).First();
return(committedOffset.Offset);
}
public IReadOnlyCollection <MessageFlowStats> GetFlowStats(IMessageFlow messageFlow)
{
var settings = _kafkaSettingsFactory.CreateReceiverSettings(messageFlow);
using var consumer = new ConsumerBuilder <Ignore, Ignore>(settings.Config).Build();
var topicPartitions = GetTopicPartitions(settings.TopicPartitionOffsets.Select(x => x.Topic));
var stats = consumer.Committed(topicPartitions, settings.PollTimeout).Select(x =>
{
var offsets = consumer.QueryWatermarkOffsets(x.TopicPartition, settings.PollTimeout);
return(new MessageFlowStats(x.TopicPartition, offsets.High, x.Offset));
}).ToList();
return(stats);
}
private void Refresh_Click(object sender, RoutedEventArgs _)
{
try
{
var config = new ConsumerConfig
{
GroupId = dataContext.GroupId,
BootstrapServers = dataContext.EndPoint,
EnableAutoCommit = false,
ApiVersionRequest = true,
Debug = "msg,broker,topic,protocol",
};
var adminConfig = new AdminClientConfig
{
ApiVersionRequest = true,
BootstrapServers = dataContext.EndPoint,
Debug = "msg,broker,topic,protocol",
};
using (var admin = new AdminClientBuilder(config).SetLogHandler((c, msg) => { _logger.Log(BrokerInfo.MapLogLevel(msg.Level), msg.Message); }).Build())
{
var meta = admin.GetMetadata(dataContext.Topic, TimeSpan.FromSeconds(10));
using (var consumer = new ConsumerBuilder <string, string>(config).SetLogHandler((c, msg) => { _logger.Log(BrokerInfo.MapLogLevel(msg.Level), msg.Message); }).Build())
{
var topicPartition = new List <TopicPartition>();
var topicMeta = meta.Topics.Find(i => i.Topic == dataContext.Topic);
foreach (var partition in topicMeta.Partitions)
{
topicPartition.Add(new TopicPartition(dataContext.Topic, partition.PartitionId));
}
var topicPartitionOffset = consumer.Committed(topicPartition, TimeSpan.FromSeconds(10));
dataContext.TopicPartionList.Clear();
foreach (var p in topicPartitionOffset)
{
dataContext.TopicPartionList.Add(new PartitionOffset
{
Partition = p.Partition,
Offset = p.Offset.Value,
Tooltip = p.Offset.ToString(),
});
}
}
}
}
catch (Exception e)
{
MessageBox.Show($"Refresh Failed, Exception:{e.Message}");
}
}
public void Consumer_Drain(string bootstrapServers)
{
LogToFile("start Consumer_Drain");
int N = 142;
var firstProduced = Util.ProduceNullStringMessages(bootstrapServers, singlePartitionTopic, 100, N);
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
EnableAutoCommit = false,
};
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
Util.ProduceNullStringMessages(bootstrapServers, topic.Name, 100, N);
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
var offsets = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(10));
Assert.Equal(0, offsets.Low);
Assert.Equal(N, offsets.High); // offsets.High is the next message to be read == the offset of last message + 1.
consumer.Commit(new[] { new TopicPartitionOffset(topic.Name, 0, new Offset(offsets.High)) });
consumer.Subscribe(topic.Name);
var cnt = 0;
while (consumer.Assignment.Count == 0)
{
Thread.Sleep(1000);
Assert.True(cnt++ < 10);
}
var committed = consumer.Committed(TimeSpan.FromSeconds(10));
Assert.Single(committed);
Assert.Equal(N, committed[0].Offset);
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Consumer_Drain");
}
public void Consumer_AutoCommit(string bootstrapServers)
{
LogToFile("start Consumer_AutoCommit");
int N = 2;
var firstProduced = Util.ProduceNullStringMessages(bootstrapServers, singlePartitionTopic, 100, N);
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
SessionTimeoutMs = 6000,
AutoCommitIntervalMs = 1000,
EnableAutoCommit = false,
EnablePartitionEof = true
};
using (var consumer =
new ConsumerBuilder <Null, string>(consumerConfig)
.SetRebalanceHandler((c, e) =>
{
if (e.IsAssignment)
{
Assert.Single(e.Partitions);
c.Assign(new TopicPartitionOffset(singlePartitionTopic, firstProduced.Partition, firstProduced.Offset));
}
})
.Build())
{
consumer.Subscribe(singlePartitionTopic);
int msgCnt = 0;
while (true)
{
var record = consumer.Consume(TimeSpan.FromMilliseconds(100));
if (record == null)
{
continue;
}
if (record.IsPartitionEOF)
{
break;
}
msgCnt += 1;
}
Assert.Equal(msgCnt, N);
Thread.Sleep(TimeSpan.FromSeconds(3));
var committed = consumer.Committed(new [] { new TopicPartition(singlePartitionTopic, 0) }, TimeSpan.FromSeconds(10));
// if this was committing, would expect the committed offset to be first committed offset + N
// (don't need to subtract 1 since the next message to be consumed is the value that is committed).
Assert.NotEqual(firstProduced.Offset + N, committed[0].Offset);
consumer.Close();
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Consumer_AutoCommit");
}
public void Consumer_Commit_Committed_Position(string bootstrapServers)
{
LogToFile("start Consumer_Commit_Committed_Position");
const int N = 8;
const int Partition = 0;
var messages = ProduceMessages(bootstrapServers, singlePartitionTopic, Partition, N);
var firstMsgOffset = messages[0].Offset;
var consumerConfig = new ConsumerConfig
{
GroupId = Guid.NewGuid().ToString(),
BootstrapServers = bootstrapServers,
EnableAutoCommit = false
};
var firstMessage = messages[0];
var lastMessage = messages[N - 1];
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset));
// Test #0 (empty cases)
consumer.Commit(new List <TopicPartitionOffset>()); // should not throw.
var committed = consumer.Committed(new List <TopicPartition>(), TimeSpan.FromSeconds(10));
Assert.Empty(committed);
var ps = consumer.Position(new List <TopicPartition>());
Assert.Empty(ps);
// Test #0.5 (invalid cases)
ps = consumer.Position(new List <TopicPartition> {
new TopicPartition("invalid-topic", 0)
});
Assert.Single(ps);
Assert.Equal(Offset.Invalid, ps[0].Offset);
Assert.Equal("invalid-topic", ps[0].Topic);
Assert.Equal(0, (int)ps[0].Partition);
Assert.Equal(new TopicPartition("invalid-topic", 0), ps[0].TopicPartition);
// Test #1
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var os = consumer.Commit();
Assert.Equal(firstMsgOffset + 1, os[0].Offset);
ps = consumer.Position(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
});
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 1, co[0].Offset);
Assert.Equal(firstMsgOffset + 1, ps[0].Offset);
// Test #2
var record2 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
os = consumer.Commit();
Assert.Equal(firstMsgOffset + 2, os[0].Offset);
ps = consumer.Position(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
});
co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 2, ps[0].Offset);
Assert.Equal(firstMsgOffset + 2, ps[0].Offset);
}
// Test #3
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Commit(new List <TopicPartitionOffset> {
new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset + 5)
});
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 5, co[0].Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var ps = consumer.Position(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
});
Assert.Equal(firstMsgOffset + 6, ps[0].Offset);
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 5, co[0].Offset);
}
// Test #4
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
consumer.Commit(new List <TopicPartitionOffset> {
new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset + 3)
});
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var ps = consumer.Position(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
});
Assert.Equal(firstMsgOffset + 4, ps[0].Offset);
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
// Test #5
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var record2 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var record3 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
consumer.Commit(record3);
var record4 = consumer.Consume(TimeSpan.FromMilliseconds(1000));
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
Assert.Equal(firstMsgOffset + 3, record.Offset);
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
// Test #6
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartitionOffset(singlePartitionTopic, 0, firstMsgOffset));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var record2 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var record3 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
consumer.Commit(record3);
var record4 = consumer.Consume(TimeSpan.FromMilliseconds(6000));
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
using (var consumer = new ConsumerBuilder <byte[], byte[]>(consumerConfig).Build())
{
consumer.Assign(new TopicPartition(singlePartitionTopic, 0));
var record = consumer.Consume(TimeSpan.FromMilliseconds(6000));
Assert.Equal(firstMsgOffset + 3, record.Offset);
var co = consumer.Committed(new List <TopicPartition> {
new TopicPartition(singlePartitionTopic, 0)
}, TimeSpan.FromSeconds(10));
Assert.Equal(firstMsgOffset + 3, co[0].Offset);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Consumer_Commit_Committed_Position");
}
public void Transactions_Commit(string bootstrapServers)
{
LogToFile("start Transactions_Commit");
var defaultTimeout = TimeSpan.FromSeconds(30);
using (var topic = new TemporaryTopic(bootstrapServers, 1))
{
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString()
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = "unimportant", EnableAutoCommit = false, Debug = "all"
}).Build())
{
var wm = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), defaultTimeout);
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, wm.High));
producer.InitTransactions(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 0", Value = "test val 0"
});
producer.CommitTransaction(defaultTimeout);
producer.BeginTransaction();
producer.Produce(topic.Name, new Message <string, string> {
Key = "test key 1", Value = "test val 1"
});
producer.CommitTransaction(defaultTimeout);
var cr1 = consumer.Consume();
var cr2 = consumer.Consume();
var cr3 = consumer.Consume(TimeSpan.FromMilliseconds(100)); // force the consumer to read over the final control message internally.
Assert.Equal(wm.High, cr1.Offset);
Assert.Equal(wm.High + 2, cr2.Offset); // there should be a skipped offset due to a commit marker in the log.
Assert.Null(cr3); // control message should not be exposed to application.
// Test that the committed offset accounts for the final ctrl message.
consumer.Commit();
}
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString()
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
BootstrapServers = bootstrapServers, GroupId = "unimportant", EnableAutoCommit = false, AutoOffsetReset = AutoOffsetReset.Latest
}).Build())
{
consumer.Assign(new TopicPartition(topic.Name, 0));
// call InitTransactions to prevent a race conidtion between a slow txn commit and a quick offset request.
producer.InitTransactions(defaultTimeout);
var committed = consumer.Committed(defaultTimeout);
var wm = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), defaultTimeout);
Assert.Equal(wm.High, committed[0].Offset);
}
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_Commit");
}