void RebalanceCallback(IntPtr rk, ErrorCode err, /* rd_kafka_topic_partition_list_t * */ IntPtr partitions, IntPtr opaque) { var partitionList = SafeKafkaHandle.GetTopicPartitionOffsetList(partitions); if (err == ErrorCode._ASSIGN_PARTITIONS) { var handler = OnPartitionsAssigned; if (handler != null && handler.GetInvocationList().Length > 0) { handler(this, partitionList); } else { Assign(partitionList); } } if (err == ErrorCode._REVOKE_PARTITIONS) { var handler = OnPartitionsRevoked; if (handler != null && handler.GetInvocationList().Length > 0) { handler(this, partitionList); } else { Unassign(); } } }
internal Topic(SafeKafkaHandle kafkaHandle, Producer producer, string topic, TopicConfig config) { this.producer = producer; config = config ?? new TopicConfig(); config["produce.offset.report"] = "true"; IntPtr configPtr = config.handle.Dup(); if (config.CustomPartitioner != null) { PartitionerDelegate = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt, IntPtr rkt_opaque, IntPtr msg_opaque) => { byte[] key = null; if (keydata != IntPtr.Zero) { key = new byte[(int)keylen]; Marshal.Copy(keydata, key, 0, (int)keylen); } return(config.CustomPartitioner(this, key, partition_cnt)); }; LibRdKafka.topic_conf_set_partitioner_cb(configPtr, PartitionerDelegate); } handle = kafkaHandle.Topic(topic, configPtr); }
/// <summary> /// Initializes a new Producer instance. /// </summary> /// <param name="config"> /// librdkafka configuration parameters (refer to https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) /// TODO: Link to confluent-kafka-dotnet page with dotnet specific parameters also (i.e. default.topic.config). /// </param> /// <param name="manualPoll"> /// If true, does not start a dedicated polling thread to trigger events or receive delivery reports - /// you must call the Poll method periodically instead. /// </param> /// <param name="disableDeliveryReports"> /// If true, disables notification of delivery reports. Note: if set to true and you use a ProduceAsync variant that return /// a Task, the Tasks will never complete. Generally you should leave this parameter as false. Set it to true for "fire and /// forget" semantics and a small boost in performance. /// </param> public Producer(IEnumerable <KeyValuePair <string, object> > config, bool manualPoll = false, bool disableDeliveryReports = false) { this.topicConfig = (IEnumerable <KeyValuePair <string, object> >)config.FirstOrDefault(prop => prop.Key == "default.topic.config").Value; this.manualPoll = manualPoll; this.disableDeliveryReports = disableDeliveryReports; var configHandle = SafeConfigHandle.Create(); config .Where(prop => prop.Key != "default.topic.config") .ToList() .ForEach((kvp) => { configHandle.Set(kvp.Key, kvp.Value.ToString()); }); IntPtr configPtr = configHandle.DangerousGetHandle(); if (!disableDeliveryReports) { LibRdKafka.conf_set_dr_msg_cb(configPtr, DeliveryReportCallback); } // TODO: provide some mechanism whereby calls to the error and log callbacks are cached until // such time as event handlers have had a chance to be registered. LibRdKafka.conf_set_error_cb(configPtr, ErrorCallback); LibRdKafka.conf_set_log_cb(configPtr, LogCallback); LibRdKafka.conf_set_stats_cb(configPtr, StatsCallback); this.kafkaHandle = SafeKafkaHandle.Create(RdKafkaType.Producer, configPtr); if (!manualPoll) { callbackCts = new CancellationTokenSource(); callbackTask = StartPollTask(callbackCts.Token); } }
internal Topic(SafeKafkaHandle kafkaHandle, Producer producer, string topic, TopicConfig config, out LibRdKafka.PartitionerCallback partitionerDelegate) { // PartitionerDelegate is an out parameter as its reference must be kept outside of Topic // it may be called after topic is GC, and it may be different for different topics // so we can't simply make it static here this.producer = producer; config = config ?? new TopicConfig(); config["produce.offset.report"] = "true"; IntPtr configPtr = config.handle.Dup(); if (config.CustomPartitioner != null) { partitionerDelegate = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt, IntPtr rkt_opaque, IntPtr msg_opaque) => { byte[] key = null; if (keydata != IntPtr.Zero) { key = new byte[(int)keylen]; Marshal.Copy(keydata, key, 0, (int)keylen); } return(config.CustomPartitioner(this, key, partition_cnt)); }; LibRdKafka.topic_conf_set_partitioner_cb(configPtr, partitionerDelegate); } else { partitionerDelegate = null; } handle = kafkaHandle.Topic(topic, configPtr); }
private List <DeleteRecordsReport> extractDeleteRecordsReports(IntPtr resultPtr) => SafeKafkaHandle.GetTopicPartitionOffsetErrorList(resultPtr) .Select(a => new DeleteRecordsReport { Topic = a.Topic, Partition = a.Partition, Offset = a.Offset, Error = a.Error }) .ToList();
internal void CommitCallback(IntPtr rk, ErrorCode err, /* rd_kafka_topic_partition_list_t * */ IntPtr offsets, IntPtr opaque) { OnOffsetCommit?.Invoke(this, new OffsetCommitArgs() { Error = err, Offsets = SafeKafkaHandle.GetTopicPartitionOffsetList(offsets) }); }
private void CommitCallback( IntPtr rk, ErrorCode err, IntPtr offsets, IntPtr opaque) { // Ensure registered handlers are never called as a side-effect of Dispose/Finalize (prevents deadlocks in common scenarios). if (kafkaHandle.IsClosed) { return; } offsetsCommittedHandler?.Invoke(new CommittedOffsets( SafeKafkaHandle.GetTopicPartitionOffsetErrorList(offsets), kafkaHandle.CreatePossiblyFatalError(err, null) )); }
internal void Init(RdKafkaType type, IntPtr config, Config.LogCallback logger) { ErrorDelegate = (IntPtr rk, ErrorCode err, string reason, IntPtr opaque) => { OnError?.Invoke(this, new ErrorArgs() { ErrorCode = err, Reason = reason }); }; LibRdKafka.conf_set_error_cb(config, ErrorDelegate); if (logger == null) { logger = ((string handle, int level, string fac, string buf) => { var now = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff"); Console.WriteLine($"{level}|{now}|{handle}|{fac}| {buf}"); }); } LogDelegate = (IntPtr rk, int level, string fac, string buf) => { // The log_cb is called very early during construction, before // SafeKafkaHandle or any of the C# wrappers are ready. // So we can't really pass rk on, just pass the rk name instead. var name = Marshal.PtrToStringAnsi(LibRdKafka.name(rk)); logger(name, level, fac, buf); }; LibRdKafka.conf_set_log_cb(config, LogDelegate); StatsDelegate = (IntPtr rk, IntPtr json, UIntPtr json_len, IntPtr opaque) => { OnStatistics?.Invoke(this, Marshal.PtrToStringAnsi(json)); return(0); }; LibRdKafka.conf_set_stats_cb(config, StatsDelegate); handle = SafeKafkaHandle.Create(type, config); callbackCts = new CancellationTokenSource(); callbackTask = StartCallbackTask(callbackCts.Token); }
internal static extern SafeTopicConfigHandle rd_kafka_default_topic_conf_dup(SafeKafkaHandle rk);
internal Producer(ProducerBuilder <TKey, TValue> builder) { var baseConfig = builder.ConstructBaseConfig(this); // TODO: Make Tasks auto complete when EnableDeliveryReportsPropertyName is set to false. // TODO: Hijack the "delivery.report.only.error" configuration parameter and add functionality to enforce that Tasks // that never complete are never created when this is set to true. this.statisticsHandler = baseConfig.statisticsHandler; this.logHandler = baseConfig.logHandler; this.errorHandler = baseConfig.errorHandler; this.oAuthBearerTokenRefreshHandler = baseConfig.oAuthBearerTokenRefreshHandler; this.partitioners = baseConfig.partitioners; var config = Confluent.Kafka.Config.ExtractCancellationDelayMaxMs(baseConfig.config, out this.cancellationDelayMaxMs); this.DeliveryReportCallback = DeliveryReportCallbackImpl; Librdkafka.Initialize(null); var modifiedConfig = Library.NameAndVersionConfig .Concat(config .Where(prop => prop.Key != ConfigPropertyNames.Producer.EnableBackgroundPoll && prop.Key != ConfigPropertyNames.Producer.EnableDeliveryReports && prop.Key != ConfigPropertyNames.Producer.DeliveryReportFields)) .ToList(); if (modifiedConfig.Where(obj => obj.Key == "delivery.report.only.error").Count() > 0) { // A managed object is kept alive over the duration of the produce request. If there is no // delivery report generated, there will be a memory leak. We could possibly support this // property by keeping track of delivery reports in managed code, but this seems like // more trouble than it's worth. throw new ArgumentException("The 'delivery.report.only.error' property is not supported by this client"); } var enableBackgroundPollObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.EnableBackgroundPoll).Value; if (enableBackgroundPollObj != null) { this.manualPoll = !bool.Parse(enableBackgroundPollObj); } var enableDeliveryReportsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.EnableDeliveryReports).Value; if (enableDeliveryReportsObj != null) { this.enableDeliveryReports = bool.Parse(enableDeliveryReportsObj); } var deliveryReportEnabledFieldsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.DeliveryReportFields).Value; if (deliveryReportEnabledFieldsObj != null) { var fields = deliveryReportEnabledFieldsObj.Replace(" ", ""); if (fields != "all") { this.enableDeliveryReportKey = false; this.enableDeliveryReportValue = false; this.enableDeliveryReportHeaders = false; this.enableDeliveryReportTimestamp = false; this.enableDeliveryReportPersistedStatus = false; if (fields != "none") { var parts = fields.Split(','); foreach (var part in parts) { switch (part) { case "key": this.enableDeliveryReportKey = true; break; case "value": this.enableDeliveryReportValue = true; break; case "timestamp": this.enableDeliveryReportTimestamp = true; break; case "headers": this.enableDeliveryReportHeaders = true; break; case "status": this.enableDeliveryReportPersistedStatus = true; break; default: throw new ArgumentException( $"Unknown delivery report field name '{part}' in config value '{ConfigPropertyNames.Producer.DeliveryReportFields}'."); } } } } } var configHandle = SafeConfigHandle.Create(); modifiedConfig.ForEach((kvp) => { if (kvp.Value == null) { throw new ArgumentNullException($"'{kvp.Key}' configuration parameter must not be null."); } configHandle.Set(kvp.Key, kvp.Value); }); IntPtr configPtr = configHandle.DangerousGetHandle(); if (enableDeliveryReports) { Librdkafka.conf_set_dr_msg_cb(configPtr, DeliveryReportCallback); } // Explicitly keep references to delegates so they are not reclaimed by the GC. errorCallbackDelegate = ErrorCallback; logCallbackDelegate = LogCallback; statisticsCallbackDelegate = StatisticsCallback; oAuthBearerTokenRefreshCallbackDelegate = OAuthBearerTokenRefreshCallback; if (errorHandler != null) { Librdkafka.conf_set_error_cb(configPtr, errorCallbackDelegate); } if (logHandler != null) { Librdkafka.conf_set_log_cb(configPtr, logCallbackDelegate); } if (statisticsHandler != null) { Librdkafka.conf_set_stats_cb(configPtr, statisticsCallbackDelegate); } if (oAuthBearerTokenRefreshHandler != null) { Librdkafka.conf_set_oauthbearer_token_refresh_cb(configPtr, oAuthBearerTokenRefreshCallbackDelegate); } this.ownedKafkaHandle = SafeKafkaHandle.Create(RdKafkaType.Producer, configPtr, this); if (this.partitioners?.Any() ?? false) { foreach (var partitioner in this.partitioners) { var topicConfigHandle = SafeTopicConfigHandle.Create(); IntPtr topicConfigPtr = topicConfigHandle.DangerousGetHandle(); Librdkafka.PartitionerDelegate partitionerDelegate = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt, IntPtr rkt_opaque, IntPtr msg_opaque) => { if (this.ownedKafkaHandle.IsClosed) { return(Partition.Any); } var topic = partitioner.Key; var providedPartitioner = partitioner.Value; return(CallCustomPartitioner(topic, providedPartitioner, keydata, keylen, partition_cnt, rkt_opaque, msg_opaque)); }; this.partitionerCallbacks.Add(partitioner.Key, partitionerDelegate); // Set partitioner on the topic_conf... Librdkafka.topic_conf_set_partitioner_cb(topicConfigPtr, partitionerDelegate); // Associate topic_conf with topic // this also caches the topic handle (and topic_conf) this.ownedKafkaHandle.getKafkaTopicHandle(partitioner.Key, topicConfigPtr); // topic_conf ownership was transferred topicConfigHandle.SetHandleAsInvalid(); } } configHandle.SetHandleAsInvalid(); // config object is no longer usable. if (!manualPoll) { callbackCts = new CancellationTokenSource(); callbackTask = StartPollTask(callbackCts.Token); } InitializeSerializers( builder.KeySerializer, builder.ValueSerializer, builder.AsyncKeySerializer, builder.AsyncValueSerializer); }
internal Producer(ProducerBuilder <TKey, TValue> builder) { var baseConfig = builder.ConstructBaseConfig(this); // TODO: Make Tasks auto complete when EnableDeliveryReportsPropertyName is set to false. // TODO: Hijack the "delivery.report.only.error" configuration parameter and add functionality to enforce that Tasks // that never complete are never created when this is set to true. this.statisticsHandler = baseConfig.statisticsHandler; this.logHandler = baseConfig.logHandler; this.errorHandler = baseConfig.errorHandler; var config = Confluent.Kafka.Config.ExtractCancellationDelayMaxMs(baseConfig.config, out this.cancellationDelayMaxMs); this.DeliveryReportCallback = DeliveryReportCallbackImpl; Librdkafka.Initialize(null); var modifiedConfig = config .Where(prop => prop.Key != ConfigPropertyNames.Producer.EnableBackgroundPoll && prop.Key != ConfigPropertyNames.Producer.EnableDeliveryReports && prop.Key != ConfigPropertyNames.Producer.DeliveryReportFields); if (modifiedConfig.Where(obj => obj.Key == "delivery.report.only.error").Count() > 0) { // A managed object is kept alive over the duration of the produce request. If there is no // delivery report generated, there will be a memory leak. We could possibly support this // property by keeping track of delivery reports in managed code, but this seems like // more trouble than it's worth. throw new ArgumentException("The 'delivery.report.only.error' property is not supported by this client"); } var enableBackgroundPollObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.EnableBackgroundPoll).Value; if (enableBackgroundPollObj != null) { this.manualPoll = !bool.Parse(enableBackgroundPollObj); } var enableDeliveryReportsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.EnableDeliveryReports).Value; if (enableDeliveryReportsObj != null) { this.enableDeliveryReports = bool.Parse(enableDeliveryReportsObj); } var deliveryReportEnabledFieldsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.DeliveryReportFields).Value; if (deliveryReportEnabledFieldsObj != null) { var fields = deliveryReportEnabledFieldsObj.Replace(" ", ""); if (fields != "all") { this.enableDeliveryReportKey = false; this.enableDeliveryReportValue = false; this.enableDeliveryReportHeaders = false; this.enableDeliveryReportTimestamp = false; this.enableDeliveryReportPersistedStatus = false; if (fields != "none") { var parts = fields.Split(','); foreach (var part in parts) { switch (part) { case "key": this.enableDeliveryReportKey = true; break; case "value": this.enableDeliveryReportValue = true; break; case "timestamp": this.enableDeliveryReportTimestamp = true; break; case "headers": this.enableDeliveryReportHeaders = true; break; case "status": this.enableDeliveryReportPersistedStatus = true; break; default: throw new ArgumentException( $"Unknown delivery report field name '{part}' in config value '{ConfigPropertyNames.Producer.DeliveryReportFields}'."); } } } } } var configHandle = SafeConfigHandle.Create(); modifiedConfig.ToList().ForEach((kvp) => { if (kvp.Value == null) { throw new ArgumentNullException($"'{kvp.Key}' configuration parameter must not be null."); } configHandle.Set(kvp.Key, kvp.Value); }); IntPtr configPtr = configHandle.DangerousGetHandle(); if (enableDeliveryReports) { Librdkafka.conf_set_dr_msg_cb(configPtr, DeliveryReportCallback); } // Explicitly keep references to delegates so they are not reclaimed by the GC. errorCallbackDelegate = ErrorCallback; logCallbackDelegate = LogCallback; statisticsCallbackDelegate = StatisticsCallback; Librdkafka.conf_set_error_cb(configPtr, errorCallbackDelegate); Librdkafka.conf_set_log_cb(configPtr, logCallbackDelegate); Librdkafka.conf_set_stats_cb(configPtr, statisticsCallbackDelegate); this.ownedKafkaHandle = SafeKafkaHandle.Create(RdKafkaType.Producer, configPtr, this); configHandle.SetHandleAsInvalid(); // config object is no longer useable. if (!manualPoll) { callbackCts = new CancellationTokenSource(); callbackTask = StartPollTask(callbackCts.Token); } InitializeSerializers( builder.KeySerializer, builder.ValueSerializer, builder.AsyncKeySerializer, builder.AsyncValueSerializer); }
/// <summary> /// Creates a new <see cref="Confluent.Kafka.ConsumerBase" /> instance. /// </summary> /// <param name="config"> /// A collection of librdkafka configuration parameters /// (refer to https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) /// and parameters specific to this client (refer to: /// <see cref="Confluent.Kafka.ConfigPropertyNames" />). /// At a minimum, 'bootstrap.servers' and 'group.id' must be /// specified. /// </param> public ConsumerBase(IEnumerable <KeyValuePair <string, string> > config) { Librdkafka.Initialize(null); config = Config.GetCancellationDelayMaxMs(config, out this.cancellationDelayMaxMs); if (config.FirstOrDefault(prop => string.Equals(prop.Key, "group.id", StringComparison.Ordinal)).Value == null) { throw new ArgumentException("'group.id' configuration parameter is required and was not specified."); } var modifiedConfig = config .Where(prop => prop.Key != ConfigPropertyNames.Consumer.ConsumeResultFields); var enabledFieldsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Consumer.ConsumeResultFields).Value; if (enabledFieldsObj != null) { var fields = enabledFieldsObj.ToString().Replace(" ", ""); if (fields != "all") { this.enableHeaderMarshaling = false; this.enableTimestampMarshaling = false; this.enableTopicNameMarshaling = false; if (fields != "none") { var parts = fields.Split(','); foreach (var part in parts) { switch (part) { case "headers": this.enableHeaderMarshaling = true; break; case "timestamp": this.enableTimestampMarshaling = true; break; case "topic": this.enableTopicNameMarshaling = true; break; default: throw new ArgumentException( $"Unexpected consume result field name '{part}' in config value '{ConfigPropertyNames.Consumer.ConsumeResultFields}'."); } } } } } var configHandle = SafeConfigHandle.Create(); modifiedConfig .ToList() .ForEach((kvp) => { if (kvp.Value == null) { throw new ArgumentException($"'{kvp.Key}' configuration parameter must not be null."); } configHandle.Set(kvp.Key, kvp.Value.ToString()); }); // Explicitly keep references to delegates so they are not reclaimed by the GC. rebalanceDelegate = RebalanceCallback; commitDelegate = CommitCallback; errorCallbackDelegate = ErrorCallback; logCallbackDelegate = LogCallback; statsCallbackDelegate = StatsCallback; IntPtr configPtr = configHandle.DangerousGetHandle(); Librdkafka.conf_set_rebalance_cb(configPtr, rebalanceDelegate); Librdkafka.conf_set_offset_commit_cb(configPtr, commitDelegate); Librdkafka.conf_set_error_cb(configPtr, errorCallbackDelegate); Librdkafka.conf_set_log_cb(configPtr, logCallbackDelegate); Librdkafka.conf_set_stats_cb(configPtr, statsCallbackDelegate); this.kafkaHandle = SafeKafkaHandle.Create(RdKafkaType.Consumer, configPtr, this); configHandle.SetHandleAsInvalid(); // config object is no longer useable. var pollSetConsumerError = kafkaHandle.PollSetConsumer(); if (pollSetConsumerError != ErrorCode.NoError) { throw new KafkaException(new Error(pollSetConsumerError, $"Failed to redirect the poll queue to consumer_poll queue: {ErrorCodeExtensions.GetReason(pollSetConsumerError)}")); } }
internal Producer(ProducerBuilder <TKey, TValue> builder) { var baseConfig = builder.ConstructBaseConfig(this); var partitioners = baseConfig.partitioners; var defaultPartitioner = baseConfig.defaultPartitioner; // TODO: Make Tasks auto complete when EnableDeliveryReportsPropertyName is set to false. // TODO: Hijack the "delivery.report.only.error" configuration parameter and add functionality to enforce that Tasks // that never complete are never created when this is set to true. this.statisticsHandler = baseConfig.statisticsHandler; this.logHandler = baseConfig.logHandler; this.errorHandler = baseConfig.errorHandler; this.oAuthBearerTokenRefreshHandler = baseConfig.oAuthBearerTokenRefreshHandler; var config = Confluent.Kafka.Config.ExtractCancellationDelayMaxMs(baseConfig.config, out this.cancellationDelayMaxMs); this.DeliveryReportCallback = DeliveryReportCallbackImpl; Librdkafka.Initialize(null); var modifiedConfig = Library.NameAndVersionConfig .Concat(config .Where(prop => prop.Key != ConfigPropertyNames.Producer.EnableBackgroundPoll && prop.Key != ConfigPropertyNames.Producer.EnableDeliveryReports && prop.Key != ConfigPropertyNames.Producer.DeliveryReportFields)) .ToList(); if (modifiedConfig.Where(obj => obj.Key == "delivery.report.only.error").Count() > 0) { // A managed object is kept alive over the duration of the produce request. If there is no // delivery report generated, there will be a memory leak. We could possibly support this // property by keeping track of delivery reports in managed code, but this seems like // more trouble than it's worth. throw new ArgumentException("The 'delivery.report.only.error' property is not supported by this client"); } var enableBackgroundPollObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.EnableBackgroundPoll).Value; if (enableBackgroundPollObj != null) { this.manualPoll = !bool.Parse(enableBackgroundPollObj); } var enableDeliveryReportsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.EnableDeliveryReports).Value; if (enableDeliveryReportsObj != null) { this.enableDeliveryReports = bool.Parse(enableDeliveryReportsObj); } var deliveryReportEnabledFieldsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.DeliveryReportFields).Value; if (deliveryReportEnabledFieldsObj != null) { var fields = deliveryReportEnabledFieldsObj.Replace(" ", ""); if (fields != "all") { this.enableDeliveryReportKey = false; this.enableDeliveryReportValue = false; this.enableDeliveryReportHeaders = false; this.enableDeliveryReportTimestamp = false; this.enableDeliveryReportPersistedStatus = false; if (fields != "none") { var parts = fields.Split(','); foreach (var part in parts) { switch (part) { case "key": this.enableDeliveryReportKey = true; break; case "value": this.enableDeliveryReportValue = true; break; case "timestamp": this.enableDeliveryReportTimestamp = true; break; case "headers": this.enableDeliveryReportHeaders = true; break; case "status": this.enableDeliveryReportPersistedStatus = true; break; default: throw new ArgumentException( $"Unknown delivery report field name '{part}' in config value '{ConfigPropertyNames.Producer.DeliveryReportFields}'."); } } } } } var configHandle = SafeConfigHandle.Create(); IntPtr configPtr = configHandle.DangerousGetHandle(); modifiedConfig.ForEach((kvp) => { if (kvp.Value == null) { throw new ArgumentNullException($"'{kvp.Key}' configuration parameter must not be null."); } configHandle.Set(kvp.Key, kvp.Value); }); if (enableDeliveryReports) { Librdkafka.conf_set_dr_msg_cb(configPtr, DeliveryReportCallback); } // Explicitly keep references to delegates so they are not reclaimed by the GC. errorCallbackDelegate = ErrorCallback; logCallbackDelegate = LogCallback; statisticsCallbackDelegate = StatisticsCallback; oAuthBearerTokenRefreshCallbackDelegate = OAuthBearerTokenRefreshCallback; if (errorHandler != null) { Librdkafka.conf_set_error_cb(configPtr, errorCallbackDelegate); } if (logHandler != null) { Librdkafka.conf_set_log_cb(configPtr, logCallbackDelegate); } if (statisticsHandler != null) { Librdkafka.conf_set_stats_cb(configPtr, statisticsCallbackDelegate); } if (oAuthBearerTokenRefreshHandler != null) { Librdkafka.conf_set_oauthbearer_token_refresh_cb(configPtr, oAuthBearerTokenRefreshCallbackDelegate); } Action <SafeTopicConfigHandle, PartitionerDelegate> addPartitionerToTopicConfig = (topicConfigHandle, partitioner) => { Librdkafka.PartitionerDelegate librdkafkaPartitioner = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt, IntPtr rkt_opaque, IntPtr msg_opaque) => { unsafe { var topicNamePtr = Librdkafka.topic_name(rkt); var topic = Util.Marshal.PtrToStringUTF8(topicNamePtr); var keyIsNull = keydata == IntPtr.Zero; var keyBytes = keyIsNull ? ReadOnlySpan <byte> .Empty : new ReadOnlySpan <byte>(keydata.ToPointer(), (int)keylen); return(partitioner(topic, partition_cnt, keyBytes, keyIsNull)); } }; this.partitionerHandles.Add(GCHandle.Alloc(librdkafkaPartitioner)); Librdkafka.topic_conf_set_partitioner_cb(topicConfigHandle.DangerousGetHandle(), librdkafkaPartitioner); }; // Configure the default custom partitioner. if (defaultPartitioner != null) { // The default topic config may have been modified by topic-level // configuraton parameters passed down from the top level config. // If that's the case, duplicate the default topic config to avoid // colobbering any already configured values. var defaultTopicConfigHandle = configHandle.GetDefaultTopicConfig(); SafeTopicConfigHandle topicConfigHandle = defaultTopicConfigHandle.DangerousGetHandle() != IntPtr.Zero ? defaultTopicConfigHandle.Duplicate() : SafeTopicConfigHandle.Create(); addPartitionerToTopicConfig(topicConfigHandle, defaultPartitioner); Librdkafka.conf_set_default_topic_conf(configPtr, topicConfigHandle.DangerousGetHandle()); } this.ownedKafkaHandle = SafeKafkaHandle.Create(RdKafkaType.Producer, configPtr, this); configHandle.SetHandleAsInvalid(); // ownership was transferred. // Per-topic partitioners. foreach (var partitioner in partitioners) { var topicConfigHandle = this.ownedKafkaHandle.DuplicateDefaultTopicConfig(); addPartitionerToTopicConfig(topicConfigHandle, partitioner.Value); this.ownedKafkaHandle.newTopic(partitioner.Key, topicConfigHandle.DangerousGetHandle()); } if (!manualPoll) { callbackCts = new CancellationTokenSource(); callbackTask = StartPollTask(callbackCts.Token); } InitializeSerializers( builder.KeySerializer, builder.ValueSerializer, builder.AsyncKeySerializer, builder.AsyncValueSerializer); }
internal Consumer(ConsumerBuilder <TKey, TValue> builder) { var baseConfig = builder.ConstructBaseConfig(this); this.statisticsHandler = baseConfig.statisticsHandler; this.logHandler = baseConfig.logHandler; this.errorHandler = baseConfig.errorHandler; this.partitionsAssignedHandler = baseConfig.partitionsAssignedHandler; this.partitionsRevokedHandler = baseConfig.partitionsRevokedHandler; this.partitionsLostHandler = baseConfig.partitionsLostHandler; this.offsetsCommittedHandler = baseConfig.offsetsCommittedHandler; this.oAuthBearerTokenRefreshHandler = baseConfig.oAuthBearerTokenRefreshHandler; this.revokedOrLostHandlerIsFunc = baseConfig.revokedOrLostHandlerIsFunc; Librdkafka.Initialize(null); var config = Confluent.Kafka.Config.ExtractCancellationDelayMaxMs(baseConfig.config, out this.cancellationDelayMaxMs); if (config.FirstOrDefault(prop => string.Equals(prop.Key, "group.id", StringComparison.Ordinal)).Value == null) { throw new ArgumentException("'group.id' configuration parameter is required and was not specified."); } var modifiedConfig = Library.NameAndVersionConfig .Concat(config.Where(prop => prop.Key != ConfigPropertyNames.Consumer.ConsumeResultFields)) .ToList(); var enabledFieldsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Consumer.ConsumeResultFields).Value; if (enabledFieldsObj != null) { var fields = enabledFieldsObj.Replace(" ", ""); if (fields != "all") { this.enableHeaderMarshaling = false; this.enableTimestampMarshaling = false; this.enableTopicNameMarshaling = false; if (fields != "none") { var parts = fields.Split(','); foreach (var part in parts) { switch (part) { case "headers": this.enableHeaderMarshaling = true; break; case "timestamp": this.enableTimestampMarshaling = true; break; case "topic": this.enableTopicNameMarshaling = true; break; default: throw new ArgumentException( $"Unexpected consume result field name '{part}' in config value '{ConfigPropertyNames.Consumer.ConsumeResultFields}'."); } } } } } var configHandle = SafeConfigHandle.Create(); modifiedConfig.ForEach((kvp) => { if (kvp.Value == null) { throw new ArgumentNullException($"'{kvp.Key}' configuration parameter must not be null."); } configHandle.Set(kvp.Key, kvp.Value); }); // Explicitly keep references to delegates so they are not reclaimed by the GC. rebalanceDelegate = RebalanceCallback; commitDelegate = CommitCallback; errorCallbackDelegate = ErrorCallback; logCallbackDelegate = LogCallback; statisticsCallbackDelegate = StatisticsCallback; oAuthBearerTokenRefreshCallbackDelegate = OAuthBearerTokenRefreshCallback; IntPtr configPtr = configHandle.DangerousGetHandle(); if (partitionsAssignedHandler != null || partitionsRevokedHandler != null || partitionsLostHandler != null) { Librdkafka.conf_set_rebalance_cb(configPtr, rebalanceDelegate); } if (offsetsCommittedHandler != null) { Librdkafka.conf_set_offset_commit_cb(configPtr, commitDelegate); } if (errorHandler != null) { Librdkafka.conf_set_error_cb(configPtr, errorCallbackDelegate); } if (logHandler != null) { Librdkafka.conf_set_log_cb(configPtr, logCallbackDelegate); } if (statisticsHandler != null) { Librdkafka.conf_set_stats_cb(configPtr, statisticsCallbackDelegate); } if (oAuthBearerTokenRefreshHandler != null) { Librdkafka.conf_set_oauthbearer_token_refresh_cb(configPtr, oAuthBearerTokenRefreshCallbackDelegate); } this.kafkaHandle = SafeKafkaHandle.Create(RdKafkaType.Consumer, configPtr, this); configHandle.SetHandleAsInvalid(); // config object is no longer useable. var pollSetConsumerError = kafkaHandle.PollSetConsumer(); if (pollSetConsumerError != ErrorCode.NoError) { throw new KafkaException(new Error(pollSetConsumerError, $"Failed to redirect the poll queue to consumer_poll queue: {ErrorCodeExtensions.GetReason(pollSetConsumerError)}")); } // setup key deserializer. if (builder.KeyDeserializer == null) { if (!defaultDeserializers.TryGetValue(typeof(TKey), out object deserializer)) { throw new InvalidOperationException( $"Key deserializer was not specified and there is no default deserializer defined for type {typeof(TKey).Name}."); } this.keyDeserializer = (IDeserializer <TKey>)deserializer; } else { this.keyDeserializer = builder.KeyDeserializer; } // setup value deserializer. if (builder.ValueDeserializer == null) { if (!defaultDeserializers.TryGetValue(typeof(TValue), out object deserializer)) { throw new InvalidOperationException( $"Value deserializer was not specified and there is no default deserializer defined for type {typeof(TValue).Name}."); } this.valueDeserializer = (IDeserializer <TValue>)deserializer; } else { this.valueDeserializer = builder.ValueDeserializer; } }
private void RebalanceCallback( IntPtr rk, ErrorCode err, IntPtr partitionsPtr, IntPtr opaque) { try { // Ensure registered handlers are never called as a side-effect of Dispose/Finalize (prevents deadlocks in common scenarios). if (kafkaHandle.IsClosed) { // The RebalanceCallback should never be invoked as a side effect of Dispose. // If for some reason flow of execution gets here, something is badly wrong. // (and we have a closed librdkafka handle that is expecting an assign call...) throw new Exception("Unexpected rebalance callback on disposed kafkaHandle"); } if (kafkaHandle.RebalanceProtocol == "COOPERATIVE" && this.revokedOrLostHandlerIsFunc) { throw new InvalidOperationException("Neither revoked nor lost partition handlers may return an updated assignment when a COOPERATIVE assignor is in use"); } var partitions = SafeKafkaHandle.GetTopicPartitionOffsetErrorList(partitionsPtr).Select(p => p.TopicPartition).ToList(); if (err == ErrorCode.Local_AssignPartitions) { if (partitionsAssignedHandler == null) { if (kafkaHandle.RebalanceProtocol == "COOPERATIVE") { IncrementalAssign(partitions.Select(p => new TopicPartitionOffset(p, Offset.Unset))); } else { Assign(partitions.Select(p => new TopicPartitionOffset(p, Offset.Unset))); } return; } lock (assignCallCountLockObj) { assignCallCount = 0; } var assignTo = partitionsAssignedHandler(partitions); lock (assignCallCountLockObj) { if (assignCallCount > 0) { throw new InvalidOperationException("(Incremental)Assign/Unassign must not be called in the partitions assigned handler"); } } if (kafkaHandle.RebalanceProtocol == "COOPERATIVE") { if (assignTo.Count() != partitions.Count()) { throw new InvalidOperationException("The partitions assigned handler must not return a different set of topic partitions than it was provided"); } var sortedPartitions = partitions.OrderBy(p => p).ToList(); var sortedAssignTo = assignTo.OrderBy(p => p.TopicPartition); var partitionsIter = sortedPartitions.GetEnumerator(); foreach (var p in sortedAssignTo) { partitionsIter.MoveNext(); if (p.TopicPartition != partitionsIter.Current) { throw new InvalidOperationException("The partitions assigned handler must not return a different set of topic partitions than it was provided"); } } IncrementalAssign(sortedAssignTo); } else { Assign(assignTo); } return; } if (err == ErrorCode.Local_RevokePartitions) { if (partitionsRevokedHandler == null && (!kafkaHandle.AssignmentLost || partitionsLostHandler == null)) { if (kafkaHandle.RebalanceProtocol == "COOPERATIVE") { IncrementalUnassign(partitions); } else { Unassign(); } return; } var assignmentWithPositions = new List <TopicPartitionOffset>(); foreach (var tp in partitions) { try { assignmentWithPositions.Add(new TopicPartitionOffset(tp, Position(tp))); } catch { assignmentWithPositions.Add(new TopicPartitionOffset(tp, Offset.Unset)); } } lock (assignCallCountLockObj) { assignCallCount = 0; } var assignTo = kafkaHandle.AssignmentLost ? (partitionsLostHandler != null ? partitionsLostHandler(assignmentWithPositions) : partitionsRevokedHandler(assignmentWithPositions)) : partitionsRevokedHandler(assignmentWithPositions); lock (assignCallCountLockObj) { if (assignCallCount > 0) { throw new InvalidOperationException("Assign/Unassign must not be called in the partitions revoked handler"); } } if (kafkaHandle.RebalanceProtocol == "COOPERATIVE") { // assignTo is always empty, not used in the COOPERATIVE case. IncrementalUnassign(partitions); } else { Unassign(); } return; } throw new KafkaException(kafkaHandle.CreatePossiblyFatalError(err, null)); } catch (Exception e) { Unassign(); handlerException = e; } }
private void RebalanceCallback( IntPtr rk, ErrorCode err, IntPtr partitions, IntPtr opaque) { try { var partitionAssignment = SafeKafkaHandle.GetTopicPartitionOffsetErrorList(partitions).Select(p => p.TopicPartition).ToList(); // Ensure registered handlers are never called as a side-effect of Dispose/Finalize (prevents deadlocks in common scenarios). if (kafkaHandle.IsClosed) { // The RebalanceCallback should never be invoked as a side effect of Dispose. // If for some reason flow of execution gets here, something is badly wrong. // (and we have a closed librdkafka handle that is expecting an assign call...) throw new Exception("Unexpected rebalance callback on disposed kafkaHandle"); } if (err == ErrorCode.Local_AssignPartitions) { if (partitionsAssignedHandler == null) { Assign(partitionAssignment.Select(p => new TopicPartitionOffset(p, Offset.Unset))); return; } lock (assignCallCountLockObj) { assignCallCount = 0; } var assignTo = partitionsAssignedHandler(partitionAssignment); lock (assignCallCountLockObj) { if (assignCallCount > 0) { throw new InvalidOperationException("Assign/Unassign must not be called in the partitions assigned handler."); } } Assign(assignTo); return; } if (err == ErrorCode.Local_RevokePartitions) { if (partitionsRevokedHandler == null) { Unassign(); return; } var assignmentWithPositions = new List <TopicPartitionOffset>(); foreach (var tp in partitionAssignment) { try { assignmentWithPositions.Add(new TopicPartitionOffset(tp, Position(tp))); } catch { assignmentWithPositions.Add(new TopicPartitionOffset(tp, Offset.Unset)); } } lock (assignCallCountLockObj) { assignCallCount = 0; } var assignTo = partitionsRevokedHandler(assignmentWithPositions); lock (assignCallCountLockObj) { if (assignCallCount > 0) { throw new InvalidOperationException("Assign/Unassign must not be called in the partitions revoked handler."); } } // This distinction is important because calling Assign whilst the consumer is being // closed (which will generally trigger this callback) is disallowed. if (assignTo.Count() > 0) { Assign(assignTo); } else { Unassign(); } return; } throw new KafkaException(kafkaHandle.CreatePossiblyFatalError(err, null)); } catch (Exception e) { handlerException = e; } }
private void RebalanceCallback( IntPtr rk, ErrorCode err, IntPtr partitions, IntPtr opaque) { var partitionList = SafeKafkaHandle.GetTopicPartitionOffsetErrorList(partitions).Select(p => p.TopicPartition).ToList(); // Ensure registered handlers are never called as a side-effect of Dispose/Finalize (prevents deadlocks in common scenarios). if (kafkaHandle.IsClosed) { // The RebalanceCallback should never be invoked as a side effect of Dispose. // If for some reason flow of execution gets here, something is badly wrong. // (and we have a closed librdkafka handle that is expecting an assign call...) throw new Exception("unexpected rebalance callback on disposed kafkaHandle"); } // Note: The contract with librdkafka requires the application to acknowledge rebalances by calling Assign. // To make the API less error prone, this is done automatically by the C# binding if required - the number // of times assign is called by user code is tracked, and if this is zero, then assign is called automatically. if (err == ErrorCode.Local_AssignPartitions) { var handler = OnPartitionsAssigned; if (handler != null && handler.GetInvocationList().Length > 0) { assignCallCount = 0; handler(this, partitionList); if (assignCallCount == 1) { return; } if (assignCallCount > 1) { throw new InvalidOperationException( $"Assign/Unassign was called {assignCallCount} times after OnPartitionsAssigned was raised. It must be called at most once."); } } Assign(partitionList.Select(p => new TopicPartitionOffset(p, Offset.Invalid))); } else if (err == ErrorCode.Local_RevokePartitions) { var handler = OnPartitionsRevoked; if (handler != null && handler.GetInvocationList().Length > 0) { assignCallCount = 0; handler(this, partitionList); if (assignCallCount == 1) { return; } if (assignCallCount > 1) { throw new InvalidOperationException( $"Assign/Unassign was called {assignCallCount} times after OnPartitionsAssigned was raised. It must be called at most once."); } } Unassign(); } else { throw new KafkaException(kafkaHandle.CreatePossiblyFatalError(err, null)); } }