public List <KafkaMessage> LoadMessages(KafkaTopic kafkaTopic) { var kafkaMessages = new List <KafkaMessage>(); var fetchResponse = connector.Fetch(kafkaTopic.TopicName, kafkaTopic.PartitionId, DefaultCorrelationId, clientId, kafkaTopic.Offset, 5000); foreach (var fetchTopic in fetchResponse.Topics) { foreach (var fetchPartition in fetchTopic.Partitions) { if (fetchPartition.MessageSets == null) { return(kafkaMessages); } foreach (var messageSet in fetchPartition.MessageSets) { var kafkaMessage = new KafkaMessage(fetchPartition.PartitionId, fetchTopic.TopicName, messageSet.Message.Payload, messageSet.MessageOffset); kafkaMessages.Add(kafkaMessage); kafkaTopic.Offset = kafkaMessage.MessageOffset; } } } return(kafkaMessages); }
private KafkaTopic TopicFromResponse(OffsetResponse offsetResponse) { string topicName = offsetResponse.Topics()[0]; int partitionId = offsetResponse.Partitions(topicName)[0]; short errorCode = offsetResponse.Errorcode(topicName, partitionId); if (errorCode != 0) { if (errorCode == 3) { throw new NoTopicException(topicName); } throw new KafkaException(errorCode); } long offset = offsetResponse.Offsets(topicName, partitionId)[0]; var kafkaTopic = new KafkaTopic(topicName, partitionId, offset); return(kafkaTopic); }
public List<KafkaMessage> LoadMessages(KafkaTopic kafkaTopic) { var kafkaMessages = new List<KafkaMessage>(); var fetchResponse = connector.Fetch(kafkaTopic.TopicName, kafkaTopic.PartitionId, DefaultCorrelationId, clientId, kafkaTopic.Offset, 5000); foreach (var fetchTopic in fetchResponse.Topics) { foreach (var fetchPartition in fetchTopic.Partitions) { if (fetchPartition.MessageSets == null) return kafkaMessages; foreach (var messageSet in fetchPartition.MessageSets) { var kafkaMessage = new KafkaMessage(fetchPartition.PartitionId, fetchTopic.TopicName, messageSet.Message.Payload, messageSet.MessageOffset); kafkaMessages.Add(kafkaMessage); kafkaTopic.Offset = kafkaMessage.MessageOffset; } } } return kafkaMessages; }
public KafkaMessageStream CreateMessageStream(string topicName, int partitionId, long offset ) { var kafkaTopic = new KafkaTopic(topicName, partitionId, offset); return new KafkaMessageStream(kafkaTopic, this); }
private KafkaTopic TopicFromResponse(OffsetResponse offsetResponse) { string topicName = offsetResponse.Topics()[0]; int partitionId = offsetResponse.Partitions(topicName)[0]; short errorCode = offsetResponse.Errorcode(topicName, partitionId); if ( errorCode != 0) { if (errorCode == 3) throw new NoTopicException(topicName); throw new KafkaException(errorCode); } long offset = offsetResponse.Offsets(topicName, partitionId)[0]; var kafkaTopic = new KafkaTopic(topicName, partitionId, offset); return kafkaTopic; }
public KafkaMessageStream(KafkaTopic topic, KafkaBusConnector busConnector) { this.busConnector = busConnector; this.topic = topic; cancel = false; }
public KafkaMessageStream CreateMessageStream(string topicName, int partitionId, long offset) { var kafkaTopic = new KafkaTopic(topicName, partitionId, offset); return(new KafkaMessageStream(kafkaTopic, this)); }