public void FetchFrom(KafkaReader reader) { if (_messageSetSize == 0) { Items = new MessageSetDetail[0]; return; } var previousPosition = reader.PositionProceeded; var items = new List <MessageSetDetail>(32); while (reader.PositionProceeded - previousPosition < _messageSetSize) { //var item = new MessageSetDetail(); //item.FetchFrom(reader); var maxBytes = _messageSetSize - (Int32)(reader.PositionProceeded - previousPosition); MessageSetDetail item; if (!MessageSetDetail.TryFetchFrom(reader, maxBytes, out item)) { break; } items.Add(item); } var restBytes = _messageSetSize - (Int32)(reader.PositionProceeded - previousPosition); if (restBytes > 0) { reader.DropBytes(restBytes); } Items = Decompress(items).ToArray(); }
public static Boolean TryFetchFrom(KafkaReader reader, Int32 maxBytes, out MessageSetDetail item) { if (maxBytes < 12) { item = null; return(false); } item = new MessageSetDetail(); item.Offset = reader.ReadInt64(); //move 8 item.MessageSize = reader.ReadInt32(); //move 4 if (item.MessageSize > maxBytes - 12) { item = null; return(false); } item.Message = new MessageSetItem(); item.Message.FetchFrom(reader); return(true); }
/*public ProduceRequest( IDictionary<String, List<Message>> messages, * Int32 partition, * AcknowlegeStrategy strategy = AcknowlegeStrategy.Written) * : base(ApiKey.ProduceRequest) { * if (messages == null || messages.Count == 0) { * return; * } * * RequiredAcks = strategy; //important * Timeout = 10; * var topicPartitions = new List<ProduceRequestTopicPartition>(); * foreach (var list in messages) { * if(list.Value.Count == 0) { * continue; * } * * var topicPartition = new ProduceRequestTopicPartition { * TopicName = list.Key, * Details = new[] { * new ProduceRequestTopicDetail { * Partition = partition, * MessageSet = list.Value.Count > 1 * ? new GZipMessageSet() : new MessageSet() * }, * } * }; * * var messageEntities = new List<MessageSetDetail>(); * foreach (var item in list.Value) { * var messageSet = new MessageSetDetail(); * messageEntities.Add(messageSet); * messageSet.Message = new MessageSetItem(); * if (item.Key != null) { * messageSet.Message.Key = Encoding.UTF8.GetBytes(item.Key); * } * messageSet.Message.Value = Encoding.UTF8.GetBytes(item.Value); * * } * topicPartition.Details[0].MessageSet.Items = messageEntities.ToArray(); * topicPartitions.Add(topicPartition); * } * TopicPartitions = topicPartitions.ToArray(); * }*/ public ProduceRequest(String topic, Int32 partition, IList <Message> messages, AcknowlegeStrategy strategy = AcknowlegeStrategy.Written, MessageCodec codec = MessageCodec.None) : base(ApiKey.ProduceRequest) { if (String.IsNullOrWhiteSpace(topic)) { throw new ArgumentOutOfRangeException("topic"); } if (messages == null || messages.Count == 0) { throw new ArgumentOutOfRangeException("messages"); } var messageEntities = new MessageSetDetail[messages.Count]; for (int i = 0; i < messageEntities.Length; i++) { var messageSet = messageEntities[i] = new MessageSetDetail(); messageSet.Message = new MessageSetItem(); //messageSet.Message.Attributes = codec; //Error when using nocompress enum if (messages[i].Key != null) { messageSet.Message.Key = Encoding.UTF8.GetBytes(messages[i].Key); } if (messages[i].Value != null) { messageSet.Message.Value = Encoding.UTF8.GetBytes(messages[i].Value); } } RequiredAcks = strategy; //important Timeout = 10; if (codec == MessageCodec.None) { TopicPartitions = new[] { new ProduceRequestTopicPartition { TopicName = topic, Details = new [] { new ProduceRequestTopicDetail { Partition = partition, MessageSet = new MessageSet { Items = messageEntities } } } } }; } else if (codec == MessageCodec.Gzip) { TopicPartitions = new[] { new ProduceRequestTopicPartition { TopicName = topic, Details = new [] { new ProduceRequestTopicDetail { Partition = partition, MessageSet = new GZipMessageSet { Items = messageEntities } } } } }; } else { throw new ArgumentOutOfRangeException("codec", String.Format("{0} not support", codec)); } }