private KafkaDataPayload EncodeProduceRequest(ProduceRequest request) { int totalCompressedBytes = 0; if (request.Payload == null) { request.Payload = new List <Payload>(); } var groupedPayloads = (from p in request.Payload group p by new { p.Topic, p.Partition, p.Codec } into tpc select tpc).ToList(); using (var message = EncodeHeader(request) .Pack(request.Acks) .Pack(request.TimeoutMS) .Pack(groupedPayloads.Count)) { foreach (var groupedPayload in groupedPayloads) { var payloads = groupedPayload.ToList(); message.Pack(groupedPayload.Key.Topic, StringPrefixEncoding.Int16) .Pack(payloads.Count) .Pack(groupedPayload.Key.Partition); switch (groupedPayload.Key.Codec) { case MessageCodec.CodecNone: message.Pack(Message.EncodeMessageSet(payloads.SelectMany(x => x.Messages))); break; case MessageCodec.CodecGzip: var compressedBytes = CreateGzipCompressedMessage(payloads.SelectMany(x => x.Messages)); Interlocked.Add(ref totalCompressedBytes, compressedBytes.CompressedAmount); message.Pack(Message.EncodeMessageSet(new[] { compressedBytes.CompressedMessage })); break; default: throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", groupedPayload.Key.Codec)); } } var result = new KafkaDataPayload { Buffer = message.Payload(), CorrelationId = request.CorrelationId, MessageCount = request.Payload.Sum(x => x.Messages.Count) }; StatisticsTracker.RecordProduceRequest(result.MessageCount, result.Buffer.Length, totalCompressedBytes); return(result); } }
private byte[] EncodeProduceRequest(ProduceRequest request) { var message = new WriteByteStream(); if (request.Payload == null) { request.Payload = new List <Payload>(); } var groupedPayloads = (from p in request.Payload group p by new { p.Topic, p.Partition, p.Codec } into tpc select tpc).ToList(); message.Pack(EncodeHeader(request)); //header message.Pack(request.Acks.ToBytes(), request.TimeoutMS.ToBytes(), groupedPayloads.Count.ToBytes()); //metadata foreach (var groupedPayload in groupedPayloads) { var payloads = groupedPayload.ToList(); message.Pack(groupedPayload.Key.Topic.ToInt16SizedBytes(), payloads.Count.ToBytes()); byte[] messageSet; switch (groupedPayload.Key.Codec) { case MessageCodec.CodecNone: messageSet = Message.EncodeMessageSet(payloads.SelectMany(x => x.Messages)); break; case MessageCodec.CodecGzip: messageSet = Message.EncodeMessageSet(CreateGzipCompressedMessage(payloads.SelectMany(x => x.Messages))); break; default: throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", groupedPayload.Key.Codec)); } message.Pack(groupedPayload.Key.Partition.ToBytes(), messageSet.Count().ToBytes(), messageSet); } //prepend final messages size and return message.Prepend(message.Length().ToBytes()); return(message.Payload()); }
private byte[] EncodeProduceRequest(ProduceRequest request) { if (request.Payload == null) { request.Payload = new List <Payload>(); } var groupedPayloads = (from p in request.Payload group p by new { p.Topic, p.Partition, p.Codec } into tpc select tpc).ToList(); using (var message = EncodeHeader(request) .Pack(request.Acks) .Pack(request.TimeoutMS) .Pack(groupedPayloads.Count)) { foreach (var groupedPayload in groupedPayloads) { var payloads = groupedPayload.ToList(); message.Pack(groupedPayload.Key.Topic, StringPrefixEncoding.Int16) .Pack(payloads.Count) .Pack(groupedPayload.Key.Partition); switch (groupedPayload.Key.Codec) { case MessageCodec.CodecNone: message.Pack(Message.EncodeMessageSet(payloads.SelectMany(x => x.Messages))); break; case MessageCodec.CodecGzip: message.Pack(Message.EncodeMessageSet(CreateGzipCompressedMessage(payloads.SelectMany(x => x.Messages)))); break; default: throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", groupedPayload.Key.Codec)); } } return(message.Payload()); } }