|
public static EncodeMessageSet ( IEnumerable |
||
| messages | IEnumerable |
The collection of messages to encode together. |
| return | byte[] | |
private KafkaDataPayload EncodeProduceRequest(ProduceRequest request)
{
int totalCompressedBytes = 0;
if (request.Payload == null)
{
request.Payload = new List <Payload>();
}
var groupedPayloads = (from p in request.Payload
group p by new
{
p.Topic,
p.Partition,
p.Codec
} into tpc
select tpc).ToList();
using (var message = EncodeHeader(request)
.Pack(request.Acks)
.Pack(request.TimeoutMS)
.Pack(groupedPayloads.Count))
{
foreach (var groupedPayload in groupedPayloads)
{
var payloads = groupedPayload.ToList();
message.Pack(groupedPayload.Key.Topic, StringPrefixEncoding.Int16)
.Pack(payloads.Count)
.Pack(groupedPayload.Key.Partition);
switch (groupedPayload.Key.Codec)
{
case MessageCodec.CodecNone:
message.Pack(Message.EncodeMessageSet(payloads.SelectMany(x => x.Messages)));
break;
case MessageCodec.CodecGzip:
var compressedBytes = CreateGzipCompressedMessage(payloads.SelectMany(x => x.Messages));
Interlocked.Add(ref totalCompressedBytes, compressedBytes.CompressedAmount);
message.Pack(Message.EncodeMessageSet(new[] { compressedBytes.CompressedMessage }));
break;
default:
throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", groupedPayload.Key.Codec));
}
}
var result = new KafkaDataPayload
{
Buffer = message.Payload(),
CorrelationId = request.CorrelationId,
MessageCount = request.Payload.Sum(x => x.Messages.Count)
};
StatisticsTracker.RecordProduceRequest(result.MessageCount, result.Buffer.Length, totalCompressedBytes);
return(result);
}
}
private IEnumerable <Message> CreateGzipCompressedMessage(IEnumerable <Message> messages)
{
var messageSet = Message.EncodeMessageSet(messages);
var gZipBytes = Compression.Zip(messageSet);
var compressedMessage = new Message
{
Attribute = (byte)(0x00 | (ProtocolConstants.AttributeCodeMask & (byte)MessageCodec.CodecGzip)),
Value = gZipBytes
};
return(new[] { compressedMessage });
}
private byte[] EncodeProduceRequest(ProduceRequest request)
{
var message = new WriteByteStream();
if (request.Payload == null)
{
request.Payload = new List <Payload>();
}
var groupedPayloads = (from p in request.Payload
group p by new
{
p.Topic,
p.Partition,
p.Codec
} into tpc
select tpc).ToList();
message.Pack(EncodeHeader(request)); //header
message.Pack(request.Acks.ToBytes(), request.TimeoutMS.ToBytes(), groupedPayloads.Count.ToBytes()); //metadata
foreach (var groupedPayload in groupedPayloads)
{
var payloads = groupedPayload.ToList();
message.Pack(groupedPayload.Key.Topic.ToInt16SizedBytes(), payloads.Count.ToBytes());
byte[] messageSet;
switch (groupedPayload.Key.Codec)
{
case MessageCodec.CodecNone:
messageSet = Message.EncodeMessageSet(payloads.SelectMany(x => x.Messages));
break;
case MessageCodec.CodecGzip:
messageSet = Message.EncodeMessageSet(CreateGzipCompressedMessage(payloads.SelectMany(x => x.Messages)));
break;
default:
throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", groupedPayload.Key.Codec));
}
message.Pack(groupedPayload.Key.Partition.ToBytes(), messageSet.Count().ToBytes(), messageSet);
}
//prepend final messages size and return
message.Prepend(message.Length().ToBytes());
return(message.Payload());
}
private byte[] EncodeProduceRequest(ProduceRequest request)
{
if (request.Payload == null)
{
request.Payload = new List <Payload>();
}
var groupedPayloads = (from p in request.Payload
group p by new
{
p.Topic,
p.Partition,
p.Codec
} into tpc
select tpc).ToList();
using (var message = EncodeHeader(request)
.Pack(request.Acks)
.Pack(request.TimeoutMS)
.Pack(groupedPayloads.Count))
{
foreach (var groupedPayload in groupedPayloads)
{
var payloads = groupedPayload.ToList();
message.Pack(groupedPayload.Key.Topic, StringPrefixEncoding.Int16)
.Pack(payloads.Count)
.Pack(groupedPayload.Key.Partition);
switch (groupedPayload.Key.Codec)
{
case MessageCodec.CodecNone:
message.Pack(Message.EncodeMessageSet(payloads.SelectMany(x => x.Messages)));
break;
case MessageCodec.CodecGzip:
message.Pack(Message.EncodeMessageSet(CreateGzipCompressedMessage(payloads.SelectMany(x => x.Messages))));
break;
default:
throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", groupedPayload.Key.Codec));
}
}
return(message.Payload());
}
}
private CompressedMessageResult CreateGzipCompressedMessage(IEnumerable <Message> messages)
{
var messageSet = Message.EncodeMessageSet(messages);
var gZipBytes = Compression.Zip(messageSet);
var compressedMessage = new Message
{
Attribute = (byte)(0x00 | (ProtocolConstants.AttributeCodeMask & (byte)MessageCodec.CodecGzip)),
Value = gZipBytes
};
return(new CompressedMessageResult
{
CompressedAmount = messageSet.Length - compressedMessage.Value.Length,
CompressedMessage = compressedMessage
});
}