コード例 #1
0
ファイル: Topic.cs プロジェクト: zhijia1122/rdkafka-dotnet
        internal Topic(SafeKafkaHandle kafkaHandle, Producer producer, string topic, TopicConfig config,
                       out LibRdKafka.PartitionerCallback partitionerDelegate)
        {
            // PartitionerDelegate is an out parameter as its reference must be kept outside of Topic
            // it may be called after topic is GC, and it may be different for different topics
            // so we can't simply make it static here
            this.producer = producer;

            config = config ?? new TopicConfig();
            config["produce.offset.report"] = "true";
            IntPtr configPtr = config.handle.Dup();

            if (config.CustomPartitioner != null)
            {
                partitionerDelegate = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt,
                                       IntPtr rkt_opaque, IntPtr msg_opaque) =>
                {
                    byte[] key = null;
                    if (keydata != IntPtr.Zero)
                    {
                        key = new byte[(int)keylen];
                        Marshal.Copy(keydata, key, 0, (int)keylen);
                    }
                    return(config.CustomPartitioner(this, key, partition_cnt));
                };
                LibRdKafka.topic_conf_set_partitioner_cb(configPtr, partitionerDelegate);
            }
            else
            {
                partitionerDelegate = null;
            }

            handle = kafkaHandle.Topic(topic, configPtr);
        }
コード例 #2
0
ファイル: Program.cs プロジェクト: yousifh/rdkafka-dotnet
        // WIP, not producing useful numbers yet. Assumes one partition.
        public static async Task<long> Consume(string broker, string topic)
        {
            long n = 0;

            var topicConfig = new TopicConfig();
            topicConfig["auto.offset.reset"] = "smallest";
            var config = new Config()
            {
                GroupId = "benchmark-consumer",
                DefaultTopicConfig = topicConfig
            };
            using (var consumer = new EventConsumer(config, broker))
            {
                var signal = new SemaphoreSlim(0, 1);

                consumer.OnMessage += (obj, msg) =>
                {
                    n += 1;
                };

                consumer.OnEndReached += (obj, end) =>
                {
                    Console.WriteLine($"End reached");
                    signal.Release();
                };

                consumer.Subscribe(new List<string>{topic});
                consumer.Start();

                await signal.WaitAsync();
                Console.WriteLine($"Shutting down");
            }

            return n;
        }
コード例 #3
0
ファイル: Topic.cs プロジェクト: liuhd1992/rdkafka-dotnet
        internal Topic(SafeKafkaHandle kafkaHandle, Producer producer, string topic, TopicConfig config)
        {
            this.producer = producer;

            config = config ?? new TopicConfig();
            config["produce.offset.report"] = "true";
            IntPtr configPtr = config.handle.Dup();

            if (config.CustomPartitioner != null)
            {
                PartitionerDelegate = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt,
                                       IntPtr rkt_opaque, IntPtr msg_opaque) =>
                {
                    byte[] key = null;
                    if (keydata != IntPtr.Zero)
                    {
                        key = new byte[(int)keylen];
                        Marshal.Copy(keydata, key, 0, (int)keylen);
                    }
                    return(config.CustomPartitioner(this, key, partition_cnt));
                };
                LibRdKafka.topic_conf_set_partitioner_cb(configPtr, PartitionerDelegate);
            }

            handle = kafkaHandle.Topic(topic, configPtr);
        }
コード例 #4
0
ファイル: Topic.cs プロジェクト: hoppity/rdkafka-dotnet
        internal Topic(SafeKafkaHandle kafkaHandle, Producer producer, string topic, TopicConfig config)
        {
            this.producer = producer;

            config = config ?? new TopicConfig();
            config["produce.offset.report"] = "true";
            IntPtr configPtr = config.handle.Dup();

            if (config.CustomPartitioner != null)
            {
                PartitionerDelegate = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt,
                        IntPtr rkt_opaque, IntPtr msg_opaque) =>
                {
                    byte[] key = null;
                    if (keydata != IntPtr.Zero)
                    {
                        key = new byte[(int) keylen];
                        Marshal.Copy(keydata, key, 0, (int) keylen);
                    }
                    return config.CustomPartitioner(this, key, partition_cnt);
                };
                LibRdKafka.topic_conf_set_partitioner_cb(configPtr, PartitionerDelegate);
            }

            handle = kafkaHandle.Topic(topic, configPtr);
        }
コード例 #5
0
ファイル: Program.cs プロジェクト: ducas/rdkafka-test
        private static Thread StartProducer(CancellationTokenSource tokenSource)
        {
            var timer = Metric.Timer("Published", Unit.Events);

            var thread = new Thread(() =>
            {
                var topicConfig = new TopicConfig();
                topicConfig["request.required.acks"] = "0"; // Don't require an ack from the broker.
                var config = new Config
                {
                    GroupId = consumerGroupName,
                    EnableAutoCommit = true,
                    StatisticsInterval = TimeSpan.FromSeconds(10),
                    DefaultTopicConfig = topicConfig
                };
                config["socket.blocking.max.ms"] = "1"; // Maximum time a broker socket operation may block.
                config["queue.buffering.max.ms"] = "1"; // Maximum time to buffer data when using async mode.

                using (var publisher = new Producer(config, brokers))
                using (var topic = publisher.Topic(topicName))
                {
                    while (!tokenSource.IsCancellationRequested)
                    {
                        Thread.Sleep(1000);
                        for (var i = 0; i < 100; i++)
                        {
                            var ticks = DateTime.UtcNow.Ticks;
                            topic.Produce(Encoding.UTF8.GetBytes(ticks.ToString()), partition: (int)(ticks % 2))
                                .ContinueWith(task =>
                                {
                                    if (task.Exception != null)
                                    {
                                        Console.WriteLine("{0}: Error publishing message - {1}", DateTime.Now.ToLongTimeString(), task.Exception);
                                        return;
                                    }

                                    timer.Record((DateTime.UtcNow.Ticks - ticks) / 10000, TimeUnit.Milliseconds);
                                    reports.Add(task.Result);
                                });
                        }
                    }
                    Console.WriteLine("Producer cancelled.");
                    Thread.CurrentThread.Abort();
                }
            });
            thread.Start();

            return thread;
        }
コード例 #6
0
ファイル: Program.cs プロジェクト: hoppity/rdkafka-dotnet
        public static void Main(string[] args)
        {
            string brokerList = args[0];
            string topicName = args[1];

            var topicConfig = new TopicConfig
            {
                CustomPartitioner = (top, key, cnt) =>
                {
                    var kt = (key != null) ? Encoding.UTF8.GetString(key, 0, key.Length) : "(null)";
                    int partition = (key?.Length ?? 0) % cnt;
                    bool available = top.PartitionAvailable(partition);
                    Console.WriteLine($"Partitioner topic: {top.Name} key: {kt} partition count: {cnt} -> {partition} {available}");
                    return partition;
                }
            };

            using (Producer producer = new Producer(brokerList))
            using (Topic topic = producer.Topic(topicName, topicConfig))
            {
                Console.WriteLine($"{producer.Name} producing on {topic.Name}. q to exit.");

                string text;
                while ((text = Console.ReadLine()) != "q")
                {
                    byte[] data = Encoding.UTF8.GetBytes(text);
                    byte[] key = null;
                    // Use the first word as the key
                    int index = text.IndexOf(" ");
                    if (index != -1)
                    {
                        key = Encoding.UTF8.GetBytes(text.Substring(0, index));
                    }

                    Task<DeliveryReport> deliveryReport = topic.Produce(data, key);
                    var unused = deliveryReport.ContinueWith(task =>
                    {
                        Console.WriteLine($"Partition: {task.Result.Partition}, Offset: {task.Result.Offset}");
                    });
                }
            }
        }
コード例 #7
0
ファイル: Producer.cs プロジェクト: zhijia1122/rdkafka-dotnet
        public Topic Topic(string topic, TopicConfig config = null)
        {
            LibRdKafka.PartitionerCallback partitionerDelegate;
            var kafkaTopic = new Topic(handle, this, topic, config, out partitionerDelegate);

            if (config?.CustomPartitioner != null)
            {
                // kafkaTopic may be GC before partitionerDelegate is called on all produced mesages
                // so we need to keep a reference to it.
                // We can't make it static in Topic as the partitioner will be different for each topic
                // we could make it in a static collection in Topic, but we can clear it when producer is closed,
                // (as it wait for all message to be produced)
                // so putting it in an instance collection allows us to free it eventually

                // it's not very effective for people creating a lot of topics
                // we should find a way to clear the list
                // when there is no more messages in queue related to the topic
                topicPartitioners.Add(partitionerDelegate);
            }
            return(kafkaTopic);
        }
コード例 #8
0
 public Topic Topic(string topic, TopicConfig config = null) => new Topic(handle, this, topic, config);
コード例 #9
0
 public Topic Topic(string topic, TopicConfig config = null)
 {
     return(new Topic(handle, this, topic, config));
 }
コード例 #10
0
ファイル: Producer.cs プロジェクト: hoppity/rdkafka-dotnet
 public Topic Topic(string topic, TopicConfig config = null)
 {
     return new Topic(handle, this, topic, config);
 }