Beispiel #1
0
        private async Task Fetch(ChannelWriter <Message <TKey, TValue> > messageWriter, string topic, PartitionMetadata partition, BrokerConnection broker, CancellationToken cancellationToken)
        {
            long offset = 0;

            while (!cancellationToken.IsCancellationRequested)
            {
                var fetch = await broker.Fetch(new FetchRequest
                {
                    ReplicaId      = -1,
                    MaxWaitTime    = 100,
                    MinBytes       = 1,
                    MaxBytes       = 64 * 1024,
                    IsolationLevel = 0,
                    SessionId      = 0,
                    SessionEpoc    = -1,
                    Topics         = new []
                    {
                        new FetchRequest.Topic
                        {
                            TopicName  = topic,
                            Partitions = new []
                            {
                                new FetchRequest.Topic.Partition
                                {
                                    PartitionId       = partition.PartitionId,
                                    CurrentLeaderEpoc = -1,
                                    FetchOffset       = offset,
                                    LogStartOffset    = -1,
                                    PartitionMaxBytes = 32 * 1024
                                }
                            }
                        }
                    }
                });

                long maxoffset = 0;
                long high      = 0;
                foreach (var r in fetch.Responses)
                {
                    foreach (var pr in r.PartitionResponses)
                    {
                        high = pr.HighWaterMark;
                        foreach (var batch in pr.RecordBatches)
                        {
                            foreach (var rec in batch.Records)
                            {
                                if (rec.Offset > maxoffset)
                                {
                                    maxoffset = rec.Offset;
                                }

                                var msg = new Message <TKey, TValue>
                                {
                                    Timestamp   = Timestamp.UnixTimestampMsToDateTime(batch.FirstTimeStamp + rec.TimeStampDelta),
                                    Topic       = r.TopicName,
                                    PartitionId = pr.PartitionId,
                                    Offset      = rec.Offset,
                                    Key         = await keyDeserializer.Deserialize(rec.Key),
                                    Value       = await valueDeserializer.Deserialize(rec.Value)
                                };

                                if (rec.Headers.Count > 0)
                                {
                                    foreach (var h in rec.Headers)
                                    {
                                        msg.AddHeader(h.Key, h.Value);
                                    }
                                }

                                await messageWriter.WriteAsync(msg, cancellationToken);
                            }
                        }
                    }
                }

                if (maxoffset > 0)
                {
                    offset = maxoffset + 1;
                }
                if (offset >= high)
                {
                    await Task.Delay(200); // linger
                }
            }
        }
Beispiel #2
0
        private async Task Fetch(ChannelWriter <RecordBatch.Record> messageWriter, string topic, PartitionMetadata partition, BrokerConnection broker, CancellationToken cancellationToken)
        {
            long offset = 0;

            while (!cancellationToken.IsCancellationRequested)
            {
                var fetch = await broker.Fetch(new FetchRequest
                {
                    ReplicaId      = -1,
                    MaxWaitTime    = 100,
                    MinBytes       = 1,
                    MaxBytes       = 64 * 1024,
                    IsolationLevel = 0,
                    SessionId      = 0,
                    SessionEpoc    = -1,
                    Topics         = new []
                    {
                        new FetchRequest.Topic
                        {
                            TopicName  = topic,
                            Partitions = new []
                            {
                                new FetchRequest.Topic.Partition
                                {
                                    PartitionId       = partition.PartitionId,
                                    CurrentLeaderEpoc = -1,
                                    FetchOffset       = offset,
                                    LogStartOffset    = -1,
                                    PartitionMaxBytes = 32 * 1024
                                }
                            }
                        }
                    }
                });

                long maxoffset = 0;
                long high      = 0;
                foreach (var r in fetch.Responses)
                {
                    foreach (var pr in r.PartitionResponses)
                    {
                        high = pr.HighWaterMark;
                        foreach (var batch in pr.RecordBatches)
                        {
                            foreach (var rec in batch.Records)
                            {
                                if (rec.Offset > maxoffset)
                                {
                                    maxoffset = rec.Offset;
                                }

                                await messageWriter.WriteAsync(rec, cancellationToken);
                            }
                        }
                    }
                }

                if (maxoffset > 0)
                {
                    offset = maxoffset + 1;
                }
                if (offset >= high)
                {
                    await Task.Delay(200); // linger
                }
            }
        }