public void Constuctor() { var wo = new WatermarkOffsets(42, 43); Assert.Equal(wo.Low, new Offset(42)); Assert.Equal(wo.High, new Offset(43)); }
public void CouldAssingConsumerForTopicWatermark() { // Arrange var topicName = new TopicName("Test"); var offset = new WatermarkOffsets(new Offset(1), new Offset(2)); var partition = new Partition(1); var watermark = new PartitionWatermark(topicName, offset, partition); var topicWatermark = new TopicWatermark(new[] { watermark }); var consumerMock = new Mock <IConsumer <object, object> >(); var consumer = consumerMock.Object; // Act var exception = Record.Exception(() => topicWatermark.AssignWithConsumer(consumer)); // Assert exception.Should().BeNull(); consumerMock.Verify(x => x.Assign(It.Is <IEnumerable <TopicPartitionOffset> >(value => value.Single().Topic == topicName.Value && value.Single().Partition == partition && value.Single().Offset == offset.High && value.Single().TopicPartition.Topic == topicName.Value && value.Single().TopicPartition.Partition == partition ) ), Times.Once); }
public void ToStringTests() { var wo = new WatermarkOffsets(42, 43); var str = wo.ToString(); Assert.True(str.Contains("42")); Assert.True(str.Contains("43")); Assert.True(str.IndexOf("43") > str.IndexOf("42")); }
public void CantCreatePartitionWatermarkWithNullWatermarkOffsets() { // Arrange var topicName = new TopicName("Test"); WatermarkOffsets offsets = null !; var partition = new Partition(1); // Act var exception = Record.Exception(() => new PartitionWatermark(topicName, offsets, partition)); // Assert exception.Should().NotBeNull().And.BeOfType <ArgumentNullException>(); }
public void AssingWithConsumerFailsOnNullConsumer() { // Arrange var topicName = new TopicName("Test"); var offsets = new WatermarkOffsets(new Offset(1), new Offset(2)); var partition = new Partition(1); var pw = new PartitionWatermark(topicName, offsets, partition); IConsumer <object, object> consumer = null !; // Act var exception = Record.Exception(() => pw.AssingWithConsumer(consumer)); // Assert exception.Should().NotBeNull().And.BeOfType <ArgumentNullException>(); }
public void IsWatermarkAchievedByFailsOnNullResult() { // Arrange var topicName = new TopicName("Test"); var offsets = new WatermarkOffsets(new Offset(1), new Offset(2)); var partition = new Partition(1); ConsumeResult <object, object> result = null !; var pw = new PartitionWatermark(topicName, offsets, partition); // Act var exception = Record.Exception(() => pw.IsWatermarkAchievedBy(result)); // Assert exception.Should().NotBeNull().And.BeOfType <ArgumentNullException>(); }
public void IsReadyToReadReactsCorrectlyOnOffsetValues(int startOffset, int endOffset, bool condition) { // Arrange var topicName = new TopicName("Test"); var offsets = new WatermarkOffsets(new Offset(startOffset), new Offset(endOffset)); var partition = new Partition(1); var pw = new PartitionWatermark(topicName, offsets, partition); var result = false; // Act var exception = Record.Exception(() => result = pw.IsReadyToRead()); // Assert exception.Should().BeNull(); result.Should().Be(condition); }
public void PartitionWatermarkCouldBeAssingToConsumer() { // Arrange var topicName = new TopicName("Test"); var offsets = new WatermarkOffsets(new Offset(1), new Offset(2)); var partition = new Partition(1); var pw = new PartitionWatermark(topicName, offsets, partition); var consumerMock = new Mock <IConsumer <object, object> >(); var consumer = consumerMock.Object; // Act var exception = Record.Exception(() => pw.AssingWithConsumer(consumer)); // Assert exception.Should().BeNull(); consumerMock.Verify(x => x.Assign(It.Is <TopicPartition>(a => a.Topic == topicName.Value && a.Partition == partition)), Times.Once); }
public void PartitionWatermarkCanBeCreated() { // Arrange var topicName = new TopicName("Test"); var offsets = new WatermarkOffsets(new Offset(1), new Offset(2)); var partition = new Partition(1); PartitionWatermark result = null !; // Act var exception = Record.Exception(() => result = new PartitionWatermark(topicName, offsets, partition)); // Assert exception.Should().BeNull(); result.Partition.Should().Be(partition); result.Offset.Should().Be(offsets); result.TopicName.Should().Be(topicName); }
/// <summary> /// Creates partition offset watermark. /// </summary> /// <param name="topicName">Name of the topic.</param> /// <param name="offset">Raw kafka offset representation.</param> /// <param name="partition">Raw kafka partition representation.</param> public PartitionWatermark(TopicName topicName, WatermarkOffsets offset, Partition partition) { if (topicName is null) { throw new ArgumentNullException(nameof(topicName)); } if (offset is null) { throw new ArgumentNullException(nameof(offset)); } _topicName = topicName; _offset = offset; _partition = partition; }
public void TopicPartitionOffsetCreatesCorrectly() { // Arrange var topicName = new TopicName("Test"); var offsets = new WatermarkOffsets(new Offset(1), new Offset(2)); var partition = new Partition(1); var pw = new PartitionWatermark(topicName, offsets, partition); TopicPartitionOffset res = null !; // Act var exception = Record.Exception(() => res = pw.CreateTopicPartitionWithHighOffset()); // Assert exception.Should().BeNull(); res.Offset.Should().Be(offsets.High); res.Topic.Should().Be(topicName.Value); res.TopicPartition.Partition.Should().Be(partition); res.TopicPartition.Topic.Should().Be(topicName.Value); }
public void IsWatermarkAchievedByReatcsOnNotRightOffset() { // Arrange var topicName = new TopicName("Test"); var offsets = new WatermarkOffsets(new Offset(1), new Offset(2)); var partition = new Partition(1); var result = new ConsumeResult <object, object>() { Offset = offsets.Low }; var pw = new PartitionWatermark(topicName, offsets, partition); var status = false; // Act var exception = Record.Exception(() => status = pw.IsWatermarkAchievedBy(result)); // Assert exception.Should().BeNull(); status.Should().BeFalse(); }
public async Task CanLoadWatermarksWithValidParamsAsync() { // Arrange var topic = new TopicName("test"); var clientMock = new Mock <IAdminClient>(); var client = clientMock.Object; var timeout = 1000; var loader = new TopicWatermarkLoader(topic, client, timeout); var consumerMock = new Mock <IConsumer <object, object> >(); IConsumer <object, object> consumerFactory() => consumerMock.Object; var adminClientPartition = new TopicPartition(topic.Value, new Partition(1)); var adminParitions = new[] { adminClientPartition }; var borkerMeta = new BrokerMetadata(1, "testHost", 1000); var partitionMeta = new PartitionMetadata(1, 1, new[] { 1 }, new[] { 1 }, null); var topicMeta = new TopicMetadata(topic.Value, new[] { partitionMeta }.ToList(), null); var meta = new Confluent.Kafka.Metadata( new[] { borkerMeta }.ToList(), new[] { topicMeta }.ToList(), 1, "test" ); clientMock.Setup(c => c.GetMetadata(topic.Value, TimeSpan.FromSeconds(timeout))).Returns(meta); var offets = new WatermarkOffsets(new Offset(1), new Offset(2)); consumerMock.Setup(x => x.QueryWatermarkOffsets(adminClientPartition, TimeSpan.FromSeconds(timeout))).Returns(offets); TopicWatermark result = null !; // Act var exception = await Record.ExceptionAsync(async() => result = await loader.LoadWatermarksAsync(consumerFactory, CancellationToken.None)); // Assert exception.Should().BeNull(); consumerMock.Verify(x => x.Close(), Times.Once); consumerMock.Verify(x => x.Dispose(), Times.Once); result.Should().NotBeNull(); var watermarks = result.Watermarks.ToList(); watermarks.Should().ContainSingle(); clientMock.Verify(c => c.GetMetadata(topic.Value, TimeSpan.FromSeconds(timeout)), Times.Once); consumerMock.Verify(x => x.QueryWatermarkOffsets(adminClientPartition, TimeSpan.FromSeconds(timeout)), Times.Once); watermarks.Single().TopicName.Should().Be(topic); watermarks.Single().Partition.Value.Should().Be(partitionMeta.PartitionId); watermarks.Single().Offset.Should().Be(offets); }
public async Task <IEnumerable <Models.KafkaMessageModel> > GetMessages(string topic, long count = -1) { List <Models.KafkaMessageModel> toReturn = new List <Models.KafkaMessageModel>(); if (!string.IsNullOrEmpty(this.BootstrapServers) && !string.IsNullOrEmpty(this.GroupId)) { ConsumerConfig config = new ConsumerConfig() { BootstrapServers = this.BootstrapServers, GroupId = this.GroupId, AutoOffsetReset = AutoOffsetReset.Latest }; await Task.Run(() => { using (IConsumer <string, string> consumer = new ConsumerBuilder <string, string>(config).Build()) { ConsumeResult <string, string> result = null; try { consumer.Subscribe(topic); while (!consumer.Assignment.Any()) { } TopicPartition tp = consumer.Assignment.FirstOrDefault(); WatermarkOffsets wo = consumer.QueryWatermarkOffsets(tp, TimeSpan.FromSeconds(this.TimeoutSeconds)); long numMessages = wo.High - wo.Low; if (count > 0 && count < numMessages) { numMessages = count; } consumer.Seek(new TopicPartitionOffset(tp, wo.High - numMessages)); do { result = consumer.Consume(TimeSpan.FromSeconds(this.TimeoutSeconds)); if (result != null) { try { toReturn.Add(new Models.KafkaMessageModel() { Topic = result.Topic, Value = JsonConvert.DeserializeObject <MessageModel>(result.Message.Value), Raw = result.Message.Value }); } catch (JsonSerializationException) { } /* We may add events in the future, and don't want to stop collecting current events if we haven't accounted for the structure */ } } while (result != null && result.TopicPartitionOffset.Offset.Value <= wo.High - 1); } catch (Exception) { } consumer.Unsubscribe(); consumer.Close(); } }); } return(toReturn); }
public void Transactions_WatermarkOffsets(string bootstrapServers) { LogToFile("start Transactions_WatermarkOffsets"); var groupName = Guid.NewGuid().ToString(); using (var topic = new TemporaryTopic(bootstrapServers, 1)) using (var producer = new ProducerBuilder <string, string>(new ProducerConfig { BootstrapServers = bootstrapServers, TransactionalId = Guid.NewGuid().ToString(), LingerMs = 0 }).Build()) using (var consumer = new ConsumerBuilder <string, string>(new ConsumerConfig { IsolationLevel = IsolationLevel.ReadCommitted, BootstrapServers = bootstrapServers, GroupId = groupName, EnableAutoCommit = false }).Build()) { var wo1 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0)); Assert.Equal(Offset.Unset, wo1.Low); Assert.Equal(Offset.Unset, wo1.High); consumer.Assign(new TopicPartitionOffset(topic.Name, 0, 0)); producer.InitTransactions(TimeSpan.FromSeconds(30)); producer.BeginTransaction(); producer.ProduceAsync(topic.Name, new Message <string, string> { Key = "test", Value = "message1" }).Wait(); producer.ProduceAsync(topic.Name, new Message <string, string> { Key = "test", Value = "message2" }).Wait(); producer.ProduceAsync(topic.Name, new Message <string, string> { Key = "test", Value = "message3" }).Wait(); WatermarkOffsets wo2 = new WatermarkOffsets(Offset.Unset, Offset.Unset); for (int i = 0; i < 10; ++i) { var cr = consumer.Consume(TimeSpan.FromMilliseconds(500)); wo2 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0)); if (wo2.High == 3) { break; } } Assert.Equal(3, wo2.High); producer.CommitTransaction(TimeSpan.FromSeconds(30)); WatermarkOffsets wo3 = new WatermarkOffsets(Offset.Unset, Offset.Unset); for (int i = 0; i < 10; ++i) { var cr2 = consumer.Consume(TimeSpan.FromSeconds(500)); wo3 = consumer.GetWatermarkOffsets(new TopicPartition(topic.Name, 0)); if (wo3.High > 3) { break; } } Assert.Equal(4, wo3.High); var wo4 = consumer.QueryWatermarkOffsets(new TopicPartition(topic.Name, 0), TimeSpan.FromSeconds(30)); Assert.Equal(4, wo4.High); } Assert.Equal(0, Library.HandleCount); LogToFile("end Transactions_WatermarkOffsets"); }
public OffsetsWatermark(WatermarkOffsets watermark) { this.watermark = watermark; }