public void FetchFrom(KafkaReader reader)
{
if (_messageSetSize == 0)
{
Items = new MessageSetDetail[0];
return;
}
var previousPosition = reader.PositionProceeded;
var items = new List <MessageSetDetail>(32);
while (reader.PositionProceeded - previousPosition < _messageSetSize)
{
//var item = new MessageSetDetail();
//item.FetchFrom(reader);
var maxBytes = _messageSetSize - (Int32)(reader.PositionProceeded - previousPosition);
MessageSetDetail item;
if (!MessageSetDetail.TryFetchFrom(reader, maxBytes, out item))
{
break;
}
items.Add(item);
}
var restBytes = _messageSetSize - (Int32)(reader.PositionProceeded - previousPosition);
if (restBytes > 0)
{
reader.DropBytes(restBytes);
}
Items = Decompress(items).ToArray();
}
public static Boolean TryFetchFrom(KafkaReader reader, Int32 maxBytes, out MessageSetDetail item)
{
if (maxBytes < 12)
{
item = null;
return(false);
}
item = new MessageSetDetail();
item.Offset = reader.ReadInt64(); //move 8
item.MessageSize = reader.ReadInt32(); //move 4
if (item.MessageSize > maxBytes - 12)
{
item = null;
return(false);
}
item.Message = new MessageSetItem();
item.Message.FetchFrom(reader);
return(true);
}
/*public ProduceRequest( IDictionary<String, List<Message>> messages,
* Int32 partition,
* AcknowlegeStrategy strategy = AcknowlegeStrategy.Written)
* : base(ApiKey.ProduceRequest) {
* if (messages == null || messages.Count == 0) {
* return;
* }
*
* RequiredAcks = strategy; //important
* Timeout = 10;
* var topicPartitions = new List<ProduceRequestTopicPartition>();
* foreach (var list in messages) {
* if(list.Value.Count == 0) {
* continue;
* }
*
* var topicPartition = new ProduceRequestTopicPartition {
* TopicName = list.Key,
* Details = new[] {
* new ProduceRequestTopicDetail {
* Partition = partition,
* MessageSet = list.Value.Count > 1
* ? new GZipMessageSet() : new MessageSet()
* },
* }
* };
*
* var messageEntities = new List<MessageSetDetail>();
* foreach (var item in list.Value) {
* var messageSet = new MessageSetDetail();
* messageEntities.Add(messageSet);
* messageSet.Message = new MessageSetItem();
* if (item.Key != null) {
* messageSet.Message.Key = Encoding.UTF8.GetBytes(item.Key);
* }
* messageSet.Message.Value = Encoding.UTF8.GetBytes(item.Value);
*
* }
* topicPartition.Details[0].MessageSet.Items = messageEntities.ToArray();
* topicPartitions.Add(topicPartition);
* }
* TopicPartitions = topicPartitions.ToArray();
* }*/
public ProduceRequest(String topic,
Int32 partition,
IList <Message> messages,
AcknowlegeStrategy strategy = AcknowlegeStrategy.Written,
MessageCodec codec = MessageCodec.None)
: base(ApiKey.ProduceRequest)
{
if (String.IsNullOrWhiteSpace(topic))
{
throw new ArgumentOutOfRangeException("topic");
}
if (messages == null || messages.Count == 0)
{
throw new ArgumentOutOfRangeException("messages");
}
var messageEntities = new MessageSetDetail[messages.Count];
for (int i = 0; i < messageEntities.Length; i++)
{
var messageSet = messageEntities[i] = new MessageSetDetail();
messageSet.Message = new MessageSetItem();
//messageSet.Message.Attributes = codec; //Error when using nocompress enum
if (messages[i].Key != null)
{
messageSet.Message.Key = Encoding.UTF8.GetBytes(messages[i].Key);
}
if (messages[i].Value != null)
{
messageSet.Message.Value = Encoding.UTF8.GetBytes(messages[i].Value);
}
}
RequiredAcks = strategy; //important
Timeout = 10;
if (codec == MessageCodec.None)
{
TopicPartitions = new[] {
new ProduceRequestTopicPartition {
TopicName = topic,
Details = new [] {
new ProduceRequestTopicDetail {
Partition = partition,
MessageSet = new MessageSet {
Items = messageEntities
}
}
}
}
};
}
else if (codec == MessageCodec.Gzip)
{
TopicPartitions = new[] {
new ProduceRequestTopicPartition {
TopicName = topic,
Details = new [] {
new ProduceRequestTopicDetail {
Partition = partition,
MessageSet = new GZipMessageSet {
Items = messageEntities
}
}
}
}
};
}
else
{
throw new ArgumentOutOfRangeException("codec", String.Format("{0} not support", codec));
}
}
