Beispiel #1
0
        public SerializingProducer(Producer producer, ISerializer <TKey> keySerializer, ISerializer <TValue> valueSerializer)
        {
            this.producer   = producer;
            KeySerializer   = keySerializer;
            ValueSerializer = valueSerializer;

            // TODO: allow serializers to be set in the producer config IEnumerable<KeyValuePair<string, object>>.

            if (KeySerializer == null)
            {
                if (typeof(TKey) != typeof(Null))
                {
                    throw new ArgumentNullException("Key serializer must be specified.");
                }
            }

            if (ValueSerializer == null)
            {
                if (typeof(TValue) != typeof(Null))
                {
                    throw new ArgumentNullException("Value serializer must be specified.");
                }
            }

            producer.OnLog        += (sender, e) => OnLog?.Invoke(sender, e);
            producer.OnError      += (sender, e) => OnError?.Invoke(sender, e);
            producer.OnStatistics += (sender, e) => OnStatistics?.Invoke(sender, e);
        }
Beispiel #2
0
 private int StatsCallback(IntPtr rk, IntPtr json, UIntPtr json_len, IntPtr opaque)
 {
     // Ensure registered handlers are never called as a side-effect of Dispose/Finalize (prevents deadlocks in common scenarios).
     if (kafkaHandle.IsClosed)
     {
         return(0);
     }
     OnStatistics?.Invoke(this, Util.Marshal.PtrToStringUTF8(json));
     return(0); // instruct librdkafka to immediately free the json ptr.
 }
        public Consumer(
            IEnumerable <KeyValuePair <string, object> > config,
            IDeserializer <TKey> keyDeserializer,
            IDeserializer <TValue> valueDeserializer)
        {
            KeyDeserializer   = keyDeserializer;
            ValueDeserializer = valueDeserializer;

            // TODO: allow deserializers to be set in the producer config IEnumerable<KeyValuePair<string, object>>.

            if (KeyDeserializer == null)
            {
                if (typeof(TKey) != typeof(Null))
                {
                    throw new ArgumentNullException("Key deserializer must be specified.");
                }
                // TKey == Null -> cast is always valid.
                KeyDeserializer = (IDeserializer <TKey>) new NullDeserializer();
            }

            if (ValueDeserializer == null)
            {
                if (typeof(TValue) != typeof(Null))
                {
                    throw new ArgumentNullException("Value deserializer must be specified.");
                }
                // TValue == Null -> cast is always valid.
                ValueDeserializer = (IDeserializer <TValue>) new NullDeserializer();
            }

            consumer                       = new Consumer(config);
            consumer.OnLog                += (sender, e) => OnLog?.Invoke(sender, e);
            consumer.OnError              += (sender, e) => OnError?.Invoke(sender, e);
            consumer.OnStatistics         += (sender, e) => OnStatistics?.Invoke(sender, e);
            consumer.OnPartitionsAssigned += (sender, e) => OnPartitionsAssigned?.Invoke(sender, e);
            consumer.OnPartitionsRevoked  += (sender, e) => OnPartitionsRevoked?.Invoke(sender, e);
            consumer.OnOffsetCommit       += (sender, e) => OnOffsetCommit?.Invoke(sender, e);
            // TODO: bypass this.consumer for this event to optimize perf.
            consumer.OnMessage += (sender, e) => OnMessage?.Invoke(sender,
                                                                   new Message <TKey, TValue> (
                                                                       e.Topic,
                                                                       e.Partition,
                                                                       e.Offset,
                                                                       KeyDeserializer.Deserialize(e.Key),
                                                                       ValueDeserializer.Deserialize(e.Value),
                                                                       e.Timestamp,
                                                                       e.Error
                                                                       )
                                                                   );
            consumer.OnPartitionEOF += (sender, e) => OnPartitionEOF?.Invoke(sender, e);
        }
Beispiel #4
0
        internal void Init(RdKafkaType type, IntPtr config, Config.LogCallback logger)
        {
            ErrorDelegate = (IntPtr rk, ErrorCode err, string reason, IntPtr opaque) =>
            {
                OnError?.Invoke(this, new ErrorArgs()
                {
                    ErrorCode = err,
                    Reason    = reason
                });
            };
            LibRdKafka.conf_set_error_cb(config, ErrorDelegate);

            if (logger == null)
            {
                logger = ((string handle, int level, string fac, string buf) =>
                {
                    var now = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff");
                    Console.WriteLine($"{level}|{now}|{handle}|{fac}| {buf}");
                });
            }

            LogDelegate = (IntPtr rk, int level, string fac, string buf) =>
            {
                // The log_cb is called very early during construction, before
                // SafeKafkaHandle or any of the C# wrappers are ready.
                // So we can't really pass rk on, just pass the rk name instead.
                var name = Marshal.PtrToStringAnsi(LibRdKafka.name(rk));
                logger(name, level, fac, buf);
            };
            LibRdKafka.conf_set_log_cb(config, LogDelegate);

            StatsDelegate = (IntPtr rk, IntPtr json, UIntPtr json_len, IntPtr opaque) =>
            {
                OnStatistics?.Invoke(this, Marshal.PtrToStringAnsi(json));
                return(0);
            };
            LibRdKafka.conf_set_stats_cb(config, StatsDelegate);

            handle = SafeKafkaHandle.Create(type, config);

            callbackCts  = new CancellationTokenSource();
            callbackTask = StartCallbackTask(callbackCts.Token);
        }
        private ConsumerBuilder <K, T> CreateKafkaConsumerBuilder()
        {
            var kafkaConsumerBuilder = new ConsumerBuilder <K, T>(consumerConfig);

            kafkaConsumerBuilder.SetKeyDeserializer(keyDeserializer);
            kafkaConsumerBuilder.SetValueDeserializer(valueDeserializer);
            kafkaConsumerBuilder.SetErrorHandler((_, e) => OnError?.Invoke(new StreamingError {
                IsFatal = e.IsFatal, Reason = e.Reason
            }));
            kafkaConsumerBuilder.SetStatisticsHandler((_, statistics) => OnStatistics?.Invoke(statistics));

            if (partitionsAssignedHandle != null && CommitEnable)
            {
                throw new ArgumentException("The partition assigned handle can not been setted if " +
                                            "'CommitEnable' property was setted to true.");
            }
            else if (partitionsAssignedHandle != null)
            {
                kafkaConsumerBuilder.SetPartitionsAssignedHandler(partitionsAssignedHandle);
            }

            return(kafkaConsumerBuilder);
        }
Beispiel #6
0
 private int StatsCallback(IntPtr rk, IntPtr json, UIntPtr json_len, IntPtr opaque)
 {
     OnStatistics?.Invoke(this, Util.Marshal.PtrToStringUTF8(json));
     return(0); // instruct librdkafka to immediately free the json ptr.
 }
        private void DumpMessages <M>(IEnumerable <M> messages, Func <M, Message <K, T> > getMessage, CancellationToken cancellationToken)
        {
            var producerBuilder = new ProducerBuilder <K, T>(producerConfig);

            producerBuilder.SetKeySerializer(keySerializer);
            producerBuilder.SetValueSerializer(valueSerializer);
            producerBuilder.SetErrorHandler((_, e) => OnError?.Invoke(new StreamingError {
                IsFatal = e.IsFatal, Reason = e.Reason
            }));
            producerBuilder.SetStatisticsHandler((_, statistics) => OnStatistics?.Invoke(statistics));

            Stopwatch processTime = Stopwatch.StartNew();

            using (var p = producerBuilder.Build())
            {
                foreach (M message in messages)
                {
                    if (cancellationToken.IsCancellationRequested)
                    {
                        break;
                    }

                    Policy.Handle <ProduceException <K, T> >()
                    .WaitAndRetryForever(retryAttempt => TimeSpan.FromMilliseconds(Math.Min(100 * Math.Pow(2, retryAttempt), 10000)),
                                         (exception, timespan) =>
                    {
                        var kafkaException = exception as ProduceException <K, T>;
                        OnError?.Invoke(new StreamingError
                        {
                            IsFatal = kafkaException.Error.IsFatal,
                            Reason  = $"{kafkaException.Error.Reason}. The message with key {kafkaException.DeliveryResult.Key} in topic {kafkaException.DeliveryResult.Topic} will be resent on {timespan.TotalMilliseconds} ms."
                        });
                    })
                    .Execute(() =>
                    {
                        if (!cancellationToken.IsCancellationRequested)
                        {
                            p.Produce(this.topic, getMessage.Invoke(message),
                                      r =>
                            {
                                if (r.Error.IsError)
                                {
                                    OnError?.Invoke(new StreamingError {
                                        IsFatal = r.Error.IsFatal, Reason = r.Error.Reason
                                    });
                                }
                            });
                        }
                    });

                    if (processTime.ElapsedMilliseconds >= CommitTimeoutMs)
                    {
                        p.Flush(cancellationToken);
                        if (!cancellationToken.IsCancellationRequested)
                        {
                            OnCommit?.Invoke();
                            processTime.Restart();
                        }
                    }
                }

                p.Flush(cancellationToken);

                if (!cancellationToken.IsCancellationRequested)
                {
                    OnCommit?.Invoke();
                }
            }
        }