public static void QueueNetworkWrite(KafkaEndpoint endpoint, KafkaDataPayload payload)
{
if (payload.TrackPayload == false) return;
var stat = new NetworkWriteStatistic(endpoint, payload);
NetworkWriteQueuedIndex.TryAdd(payload.CorrelationId, stat);
Interlocked.Increment(ref Gauges.QueuedWriteOperation);
}
private KafkaDataPayload EncodeProduceRequest(ProduceRequest request)
{
int totalCompressedBytes = 0;
if (request.Payload == null)
{
request.Payload = new List <Payload>();
}
var groupedPayloads = (from p in request.Payload
group p by new
{
p.Topic,
p.Partition,
p.Codec
} into tpc
select tpc).ToList();
using (var message = EncodeHeader(request)
.Pack(request.Acks)
.Pack(request.TimeoutMS)
.Pack(groupedPayloads.Count))
{
foreach (var groupedPayload in groupedPayloads)
{
var payloads = groupedPayload.ToList();
message.Pack(groupedPayload.Key.Topic, StringPrefixEncoding.Int16)
.Pack(payloads.Count)
.Pack(groupedPayload.Key.Partition);
switch (groupedPayload.Key.Codec)
{
case MessageCodec.CodecNone:
message.Pack(Message.EncodeMessageSet(payloads.SelectMany(x => x.Messages)));
break;
case MessageCodec.CodecGzip:
var compressedBytes = CreateGzipCompressedMessage(payloads.SelectMany(x => x.Messages));
Interlocked.Add(ref totalCompressedBytes, compressedBytes.CompressedAmount);
message.Pack(Message.EncodeMessageSet(new[] { compressedBytes.CompressedMessage }));
break;
default:
throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", groupedPayload.Key.Codec));
}
}
var result = new KafkaDataPayload
{
Buffer = message.Payload(),
CorrelationId = request.CorrelationId,
MessageCount = request.Payload.Sum(x => x.Messages.Count)
};
StatisticsTracker.RecordProduceRequest(result.MessageCount, result.Buffer.Length, totalCompressedBytes);
return(result);
}
}
public static void QueueNetworkWrite(KafkaEndpoint endpoint, KafkaDataPayload payload)
{
if (payload.TrackPayload == false)
{
return;
}
var stat = new NetworkWriteStatistic(endpoint, payload);
NetworkWriteQueuedIndex.TryAdd(payload.CorrelationId, stat);
Interlocked.Increment(ref Gauges.QueuedWriteOperation);
}
public static void CompleteNetworkWrite(KafkaDataPayload payload, long milliseconds, bool failed)
{
if (payload.TrackPayload == false)
{
return;
}
NetworkWriteStatistic stat;
if (NetworkWriteQueuedIndex.TryRemove(payload.CorrelationId, out stat))
{
stat.SetCompleted(milliseconds, failed);
CompletedNetworkWriteStatistics.Enqueue(stat);
}
Interlocked.Decrement(ref Gauges.QueuedWriteOperation);
}
public Task SendAsync(KafkaDataPayload payload)
{
throw new NotImplementedException();
}
public NetworkWriteStatistic(KafkaEndpoint endpoint, KafkaDataPayload payload)
{
CreatedOnUtc = DateTime.UtcNow;
Endpoint = endpoint;
Payload = payload;
}
public static void CompleteNetworkWrite(KafkaDataPayload payload, long milliseconds, bool failed)
{
if (payload.TrackPayload == false) return;
NetworkWriteStatistic stat;
if (NetworkWriteQueuedIndex.TryRemove(payload.CorrelationId, out stat))
{
stat.SetCompleted(milliseconds, failed);
CompletedNetworkWriteStatistics.Enqueue(stat);
}
Interlocked.Decrement(ref Gauges.QueuedWriteOperation);
}
public NetworkWriteStatistic(KafkaEndpoint endpoint, KafkaDataPayload payload)
{
CreatedOnUtc = DateTime.UtcNow;
Endpoint = endpoint;
Payload = payload;
}