private async Task Fetch(ChannelWriter <Message <TKey, TValue> > messageWriter, string topic, PartitionMetadata partition, BrokerConnection broker, CancellationToken cancellationToken)
{
long offset = 0;
while (!cancellationToken.IsCancellationRequested)
{
var fetch = await broker.Fetch(new FetchRequest
{
ReplicaId = -1,
MaxWaitTime = 100,
MinBytes = 1,
MaxBytes = 64 * 1024,
IsolationLevel = 0,
SessionId = 0,
SessionEpoc = -1,
Topics = new []
{
new FetchRequest.Topic
{
TopicName = topic,
Partitions = new []
{
new FetchRequest.Topic.Partition
{
PartitionId = partition.PartitionId,
CurrentLeaderEpoc = -1,
FetchOffset = offset,
LogStartOffset = -1,
PartitionMaxBytes = 32 * 1024
}
}
}
}
});
long maxoffset = 0;
long high = 0;
foreach (var r in fetch.Responses)
{
foreach (var pr in r.PartitionResponses)
{
high = pr.HighWaterMark;
foreach (var batch in pr.RecordBatches)
{
foreach (var rec in batch.Records)
{
if (rec.Offset > maxoffset)
{
maxoffset = rec.Offset;
}
var msg = new Message <TKey, TValue>
{
Timestamp = Timestamp.UnixTimestampMsToDateTime(batch.FirstTimeStamp + rec.TimeStampDelta),
Topic = r.TopicName,
PartitionId = pr.PartitionId,
Offset = rec.Offset,
Key = await keyDeserializer.Deserialize(rec.Key),
Value = await valueDeserializer.Deserialize(rec.Value)
};
if (rec.Headers.Count > 0)
{
foreach (var h in rec.Headers)
{
msg.AddHeader(h.Key, h.Value);
}
}
await messageWriter.WriteAsync(msg, cancellationToken);
}
}
}
}
if (maxoffset > 0)
{
offset = maxoffset + 1;
}
if (offset >= high)
{
await Task.Delay(200); // linger
}
}
}
private async Task Fetch(ChannelWriter <RecordBatch.Record> messageWriter, string topic, PartitionMetadata partition, BrokerConnection broker, CancellationToken cancellationToken)
{
long offset = 0;
while (!cancellationToken.IsCancellationRequested)
{
var fetch = await broker.Fetch(new FetchRequest
{
ReplicaId = -1,
MaxWaitTime = 100,
MinBytes = 1,
MaxBytes = 64 * 1024,
IsolationLevel = 0,
SessionId = 0,
SessionEpoc = -1,
Topics = new []
{
new FetchRequest.Topic
{
TopicName = topic,
Partitions = new []
{
new FetchRequest.Topic.Partition
{
PartitionId = partition.PartitionId,
CurrentLeaderEpoc = -1,
FetchOffset = offset,
LogStartOffset = -1,
PartitionMaxBytes = 32 * 1024
}
}
}
}
});
long maxoffset = 0;
long high = 0;
foreach (var r in fetch.Responses)
{
foreach (var pr in r.PartitionResponses)
{
high = pr.HighWaterMark;
foreach (var batch in pr.RecordBatches)
{
foreach (var rec in batch.Records)
{
if (rec.Offset > maxoffset)
{
maxoffset = rec.Offset;
}
await messageWriter.WriteAsync(rec, cancellationToken);
}
}
}
}
if (maxoffset > 0)
{
offset = maxoffset + 1;
}
if (offset >= high)
{
await Task.Delay(200); // linger
}
}
}
