public void Constuctor()
{
var wo = new WatermarkOffsets(42, 43);
Assert.Equal(wo.Low, new Offset(42));
Assert.Equal(wo.High, new Offset(43));
}
public void CouldAssingConsumerForTopicWatermark()
{
// Arrange
var topicName = new TopicName("Test");
var offset = new WatermarkOffsets(new Offset(1), new Offset(2));
var partition = new Partition(1);
var watermark = new PartitionWatermark(topicName, offset, partition);
var topicWatermark = new TopicWatermark(new[] { watermark });
var consumerMock = new Mock <IConsumer <object, object> >();
var consumer = consumerMock.Object;
// Act
var exception = Record.Exception(() => topicWatermark.AssignWithConsumer(consumer));
// Assert
exception.Should().BeNull();
consumerMock.Verify(x => x.Assign(It.Is <IEnumerable <TopicPartitionOffset> >(value =>
value.Single().Topic == topicName.Value
&&
value.Single().Partition == partition
&&
value.Single().Offset == offset.High
&&
value.Single().TopicPartition.Topic == topicName.Value
&&
value.Single().TopicPartition.Partition == partition
)
), Times.Once);
}
public void ToStringTests()
{
var wo = new WatermarkOffsets(42, 43);
var str = wo.ToString();
Assert.True(str.Contains("42"));
Assert.True(str.Contains("43"));
Assert.True(str.IndexOf("43") > str.IndexOf("42"));
}
public void CantCreatePartitionWatermarkWithNullWatermarkOffsets()
{
// Arrange
var topicName = new TopicName("Test");
WatermarkOffsets offsets = null !;
var partition = new Partition(1);
// Act
var exception = Record.Exception(() => new PartitionWatermark(topicName, offsets, partition));
// Assert
exception.Should().NotBeNull().And.BeOfType <ArgumentNullException>();
}
public void AssingWithConsumerFailsOnNullConsumer()
{
// Arrange
var topicName = new TopicName("Test");
var offsets = new WatermarkOffsets(new Offset(1), new Offset(2));
var partition = new Partition(1);
var pw = new PartitionWatermark(topicName, offsets, partition);
IConsumer <object, object> consumer = null !;
// Act
var exception = Record.Exception(() => pw.AssingWithConsumer(consumer));
// Assert
exception.Should().NotBeNull().And.BeOfType <ArgumentNullException>();
}
public void IsWatermarkAchievedByFailsOnNullResult()
{
// Arrange
var topicName = new TopicName("Test");
var offsets = new WatermarkOffsets(new Offset(1), new Offset(2));
var partition = new Partition(1);
ConsumeResult <object, object> result = null !;
var pw = new PartitionWatermark(topicName, offsets, partition);
// Act
var exception = Record.Exception(() => pw.IsWatermarkAchievedBy(result));
// Assert
exception.Should().NotBeNull().And.BeOfType <ArgumentNullException>();
}
public void IsReadyToReadReactsCorrectlyOnOffsetValues(int startOffset, int endOffset, bool condition)
{
// Arrange
var topicName = new TopicName("Test");
var offsets = new WatermarkOffsets(new Offset(startOffset), new Offset(endOffset));
var partition = new Partition(1);
var pw = new PartitionWatermark(topicName, offsets, partition);
var result = false;
// Act
var exception = Record.Exception(() => result = pw.IsReadyToRead());
// Assert
exception.Should().BeNull();
result.Should().Be(condition);
}
public void PartitionWatermarkCouldBeAssingToConsumer()
{
// Arrange
var topicName = new TopicName("Test");
var offsets = new WatermarkOffsets(new Offset(1), new Offset(2));
var partition = new Partition(1);
var pw = new PartitionWatermark(topicName, offsets, partition);
var consumerMock = new Mock <IConsumer <object, object> >();
var consumer = consumerMock.Object;
// Act
var exception = Record.Exception(() => pw.AssingWithConsumer(consumer));
// Assert
exception.Should().BeNull();
consumerMock.Verify(x => x.Assign(It.Is <TopicPartition>(a => a.Topic == topicName.Value && a.Partition == partition)), Times.Once);
}
public void PartitionWatermarkCanBeCreated()
{
// Arrange
var topicName = new TopicName("Test");
var offsets = new WatermarkOffsets(new Offset(1), new Offset(2));
var partition = new Partition(1);
PartitionWatermark result = null !;
// Act
var exception = Record.Exception(() => result = new PartitionWatermark(topicName, offsets, partition));
// Assert
exception.Should().BeNull();
result.Partition.Should().Be(partition);
result.Offset.Should().Be(offsets);
result.TopicName.Should().Be(topicName);
}
/// <summary>
/// Creates partition offset watermark.
/// </summary>
/// <param name="topicName">Name of the topic.</param>
/// <param name="offset">Raw kafka offset representation.</param>
/// <param name="partition">Raw kafka partition representation.</param>
public PartitionWatermark(TopicName topicName,
WatermarkOffsets offset,
Partition partition)
{
if (topicName is null)
{
throw new ArgumentNullException(nameof(topicName));
}
if (offset is null)
{
throw new ArgumentNullException(nameof(offset));
}
_topicName = topicName;
_offset = offset;
_partition = partition;
}
public void TopicPartitionOffsetCreatesCorrectly()
{
// Arrange
var topicName = new TopicName("Test");
var offsets = new WatermarkOffsets(new Offset(1), new Offset(2));
var partition = new Partition(1);
var pw = new PartitionWatermark(topicName, offsets, partition);
TopicPartitionOffset res = null !;
// Act
var exception = Record.Exception(() => res = pw.CreateTopicPartitionWithHighOffset());
// Assert
exception.Should().BeNull();
res.Offset.Should().Be(offsets.High);
res.Topic.Should().Be(topicName.Value);
res.TopicPartition.Partition.Should().Be(partition);
res.TopicPartition.Topic.Should().Be(topicName.Value);
}
public void IsWatermarkAchievedByReatcsOnNotRightOffset()
{
// Arrange
var topicName = new TopicName("Test");
var offsets = new WatermarkOffsets(new Offset(1), new Offset(2));
var partition = new Partition(1);
var result = new ConsumeResult <object, object>()
{
Offset = offsets.Low
};
var pw = new PartitionWatermark(topicName, offsets, partition);
var status = false;
// Act
var exception = Record.Exception(() => status = pw.IsWatermarkAchievedBy(result));
// Assert
exception.Should().BeNull();
status.Should().BeFalse();
}
public async Task CanLoadWatermarksWithValidParamsAsync()
{
// Arrange
var topic = new TopicName("test");
var clientMock = new Mock <IAdminClient>();
var client = clientMock.Object;
var timeout = 1000;
var loader = new TopicWatermarkLoader(topic, client, timeout);
var consumerMock = new Mock <IConsumer <object, object> >();
IConsumer <object, object> consumerFactory() => consumerMock.Object;
var adminClientPartition = new TopicPartition(topic.Value, new Partition(1));
var adminParitions = new[] { adminClientPartition };
var borkerMeta = new BrokerMetadata(1, "testHost", 1000);
var partitionMeta = new PartitionMetadata(1, 1, new[] { 1 }, new[] { 1 }, null);
var topicMeta = new TopicMetadata(topic.Value, new[] { partitionMeta }.ToList(), null);
var meta = new Confluent.Kafka.Metadata(
new[] { borkerMeta }.ToList(),
new[] { topicMeta }.ToList(), 1, "test"
);
clientMock.Setup(c => c.GetMetadata(topic.Value, TimeSpan.FromSeconds(timeout))).Returns(meta);
var offets = new WatermarkOffsets(new Offset(1), new Offset(2));
consumerMock.Setup(x => x.QueryWatermarkOffsets(adminClientPartition, TimeSpan.FromSeconds(timeout))).Returns(offets);
TopicWatermark result = null !;
// Act
var exception = await Record.ExceptionAsync(async() => result = await loader.LoadWatermarksAsync(consumerFactory, CancellationToken.None));
// Assert
exception.Should().BeNull();
consumerMock.Verify(x => x.Close(), Times.Once);
consumerMock.Verify(x => x.Dispose(), Times.Once);
result.Should().NotBeNull();
var watermarks = result.Watermarks.ToList();
watermarks.Should().ContainSingle();
clientMock.Verify(c => c.GetMetadata(topic.Value, TimeSpan.FromSeconds(timeout)), Times.Once);
consumerMock.Verify(x => x.QueryWatermarkOffsets(adminClientPartition, TimeSpan.FromSeconds(timeout)), Times.Once);
watermarks.Single().TopicName.Should().Be(topic);
watermarks.Single().Partition.Value.Should().Be(partitionMeta.PartitionId);
watermarks.Single().Offset.Should().Be(offets);
}
public async Task <IEnumerable <Models.KafkaMessageModel> > GetMessages(string topic, long count = -1)
{
List <Models.KafkaMessageModel> toReturn = new List <Models.KafkaMessageModel>();
if (!string.IsNullOrEmpty(this.BootstrapServers) && !string.IsNullOrEmpty(this.GroupId))
{
ConsumerConfig config = new ConsumerConfig()
{
BootstrapServers = this.BootstrapServers,
GroupId = this.GroupId,
AutoOffsetReset = AutoOffsetReset.Latest
};
await Task.Run(() =>
{
using (IConsumer <string, string> consumer = new ConsumerBuilder <string, string>(config).Build())
{
ConsumeResult <string, string> result = null;
try
{
consumer.Subscribe(topic);
while (!consumer.Assignment.Any())
{
}
TopicPartition tp = consumer.Assignment.FirstOrDefault();
WatermarkOffsets wo = consumer.QueryWatermarkOffsets(tp, TimeSpan.FromSeconds(this.TimeoutSeconds));
long numMessages = wo.High - wo.Low;
if (count > 0 && count < numMessages)
{
numMessages = count;
}
consumer.Seek(new TopicPartitionOffset(tp, wo.High - numMessages));
do
{
result = consumer.Consume(TimeSpan.FromSeconds(this.TimeoutSeconds));
if (result != null)
{
try
{
toReturn.Add(new Models.KafkaMessageModel()
{
Topic = result.Topic,
Value = JsonConvert.DeserializeObject <MessageModel>(result.Message.Value),
Raw = result.Message.Value
});
}
catch (JsonSerializationException) { } /* We may add events in the future, and don't want to stop collecting current events if we haven't accounted for the structure */
}
} while (result != null && result.TopicPartitionOffset.Offset.Value <= wo.High - 1);
}
catch (Exception) { }
consumer.Unsubscribe();
consumer.Close();
}
});
}
return(toReturn);
}
public void Transactions_WatermarkOffsets(string bootstrapServers)
{
LogToFile("start Transactions_WatermarkOffsets");
var groupName = Guid.NewGuid().ToString();
using (var topic = new TemporaryTopic(bootstrapServers, 1))
using (var producer = new ProducerBuilder <string, string>(new ProducerConfig {
BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString(), LingerMs = 0
}).Build())
using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig {
IsolationLevel = IsolationLevel.ReadCommitted, BootstrapServers = bootstrapServers, GroupId = groupName, EnableAutoCommit = false
}).Build())
{
var wo1 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
Assert.Equal(Offset.Unset, wo1.Low);
Assert.Equal(Offset.Unset, wo1.High);
consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0));
producer.InitTransactions(TimeSpan.FromSeconds(30));
producer.BeginTransaction();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message1"
}).Wait();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message2"
}).Wait();
producer.ProduceAsync(topic.Name, new Message <string, string> {
Key = "test", Value = "message3"
}).Wait();
WatermarkOffsets wo2 = new WatermarkOffsets(Offset.Unset, Offset.Unset);
for (int i = 0; i < 10; ++i)
{
var cr = consumer.Consume(TimeSpan.FromMilliseconds(500));
wo2 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
if (wo2.High == 3)
{
break;
}
}
Assert.Equal(3, wo2.High);
producer.CommitTransaction(TimeSpan.FromSeconds(30));
WatermarkOffsets wo3 = new WatermarkOffsets(Offset.Unset, Offset.Unset);
for (int i = 0; i < 10; ++i)
{
var cr2 = consumer.Consume(TimeSpan.FromSeconds(500));
wo3 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0));
if (wo3.High > 3)
{
break;
}
}
Assert.Equal(4, wo3.High);
var wo4 = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(30));
Assert.Equal(4, wo4.High);
}
Assert.Equal(0, Library.HandleCount);
LogToFile("end Transactions_WatermarkOffsets");
}
public OffsetsWatermark(WatermarkOffsets watermark)
{
this.watermark = watermark;
}
