void RebalanceCallback(IntPtr rk, ErrorCode err,
/* rd_kafka_topic_partition_list_t * */ IntPtr partitions,
IntPtr opaque)
{
var partitionList = SafeKafkaHandle.GetTopicPartitionOffsetList(partitions);
if (err == ErrorCode._ASSIGN_PARTITIONS)
{
var handler = OnPartitionsAssigned;
if (handler != null && handler.GetInvocationList().Length > 0)
{
handler(this, partitionList);
}
else
{
Assign(partitionList);
}
}
if (err == ErrorCode._REVOKE_PARTITIONS)
{
var handler = OnPartitionsRevoked;
if (handler != null && handler.GetInvocationList().Length > 0)
{
handler(this, partitionList);
}
else
{
Unassign();
}
}
}
internal Topic(SafeKafkaHandle kafkaHandle, Producer producer, string topic, TopicConfig config)
{
this.producer = producer;
config = config ?? new TopicConfig();
config["produce.offset.report"] = "true";
IntPtr configPtr = config.handle.Dup();
if (config.CustomPartitioner != null)
{
PartitionerDelegate = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt,
IntPtr rkt_opaque, IntPtr msg_opaque) =>
{
byte[] key = null;
if (keydata != IntPtr.Zero)
{
key = new byte[(int)keylen];
Marshal.Copy(keydata, key, 0, (int)keylen);
}
return(config.CustomPartitioner(this, key, partition_cnt));
};
LibRdKafka.topic_conf_set_partitioner_cb(configPtr, PartitionerDelegate);
}
handle = kafkaHandle.Topic(topic, configPtr);
}
/// <summary>
/// Initializes a new Producer instance.
/// </summary>
/// <param name="config">
/// librdkafka configuration parameters (refer to https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md)
/// TODO: Link to confluent-kafka-dotnet page with dotnet specific parameters also (i.e. default.topic.config).
/// </param>
/// <param name="manualPoll">
/// If true, does not start a dedicated polling thread to trigger events or receive delivery reports -
/// you must call the Poll method periodically instead.
/// </param>
/// <param name="disableDeliveryReports">
/// If true, disables notification of delivery reports. Note: if set to true and you use a ProduceAsync variant that return
/// a Task, the Tasks will never complete. Generally you should leave this parameter as false. Set it to true for "fire and
/// forget" semantics and a small boost in performance.
/// </param>
public Producer(IEnumerable <KeyValuePair <string, object> > config, bool manualPoll = false, bool disableDeliveryReports = false)
{
this.topicConfig = (IEnumerable <KeyValuePair <string, object> >)config.FirstOrDefault(prop => prop.Key == "default.topic.config").Value;
this.manualPoll = manualPoll;
this.disableDeliveryReports = disableDeliveryReports;
var configHandle = SafeConfigHandle.Create();
config
.Where(prop => prop.Key != "default.topic.config")
.ToList()
.ForEach((kvp) => { configHandle.Set(kvp.Key, kvp.Value.ToString()); });
IntPtr configPtr = configHandle.DangerousGetHandle();
if (!disableDeliveryReports)
{
LibRdKafka.conf_set_dr_msg_cb(configPtr, DeliveryReportCallback);
}
// TODO: provide some mechanism whereby calls to the error and log callbacks are cached until
// such time as event handlers have had a chance to be registered.
LibRdKafka.conf_set_error_cb(configPtr, ErrorCallback);
LibRdKafka.conf_set_log_cb(configPtr, LogCallback);
LibRdKafka.conf_set_stats_cb(configPtr, StatsCallback);
this.kafkaHandle = SafeKafkaHandle.Create(RdKafkaType.Producer, configPtr);
if (!manualPoll)
{
callbackCts = new CancellationTokenSource();
callbackTask = StartPollTask(callbackCts.Token);
}
}
internal Topic(SafeKafkaHandle kafkaHandle, Producer producer, string topic, TopicConfig config,
out LibRdKafka.PartitionerCallback partitionerDelegate)
{
// PartitionerDelegate is an out parameter as its reference must be kept outside of Topic
// it may be called after topic is GC, and it may be different for different topics
// so we can't simply make it static here
this.producer = producer;
config = config ?? new TopicConfig();
config["produce.offset.report"] = "true";
IntPtr configPtr = config.handle.Dup();
if (config.CustomPartitioner != null)
{
partitionerDelegate = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt,
IntPtr rkt_opaque, IntPtr msg_opaque) =>
{
byte[] key = null;
if (keydata != IntPtr.Zero)
{
key = new byte[(int)keylen];
Marshal.Copy(keydata, key, 0, (int)keylen);
}
return(config.CustomPartitioner(this, key, partition_cnt));
};
LibRdKafka.topic_conf_set_partitioner_cb(configPtr, partitionerDelegate);
}
else
{
partitionerDelegate = null;
}
handle = kafkaHandle.Topic(topic, configPtr);
}
private List <DeleteRecordsReport> extractDeleteRecordsReports(IntPtr resultPtr)
=> SafeKafkaHandle.GetTopicPartitionOffsetErrorList(resultPtr)
.Select(a => new DeleteRecordsReport
{
Topic = a.Topic,
Partition = a.Partition,
Offset = a.Offset,
Error = a.Error
})
.ToList();
internal void CommitCallback(IntPtr rk,
ErrorCode err,
/* rd_kafka_topic_partition_list_t * */ IntPtr offsets,
IntPtr opaque)
{
OnOffsetCommit?.Invoke(this, new OffsetCommitArgs()
{
Error = err,
Offsets = SafeKafkaHandle.GetTopicPartitionOffsetList(offsets)
});
}
private void CommitCallback(
IntPtr rk,
ErrorCode err,
IntPtr offsets,
IntPtr opaque)
{
// Ensure registered handlers are never called as a side-effect of Dispose/Finalize (prevents deadlocks in common scenarios).
if (kafkaHandle.IsClosed)
{
return;
}
offsetsCommittedHandler?.Invoke(new CommittedOffsets(
SafeKafkaHandle.GetTopicPartitionOffsetErrorList(offsets),
kafkaHandle.CreatePossiblyFatalError(err, null)
));
}
internal void Init(RdKafkaType type, IntPtr config, Config.LogCallback logger)
{
ErrorDelegate = (IntPtr rk, ErrorCode err, string reason, IntPtr opaque) =>
{
OnError?.Invoke(this, new ErrorArgs()
{
ErrorCode = err,
Reason = reason
});
};
LibRdKafka.conf_set_error_cb(config, ErrorDelegate);
if (logger == null)
{
logger = ((string handle, int level, string fac, string buf) =>
{
var now = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff");
Console.WriteLine($"{level}|{now}|{handle}|{fac}| {buf}");
});
}
LogDelegate = (IntPtr rk, int level, string fac, string buf) =>
{
// The log_cb is called very early during construction, before
// SafeKafkaHandle or any of the C# wrappers are ready.
// So we can't really pass rk on, just pass the rk name instead.
var name = Marshal.PtrToStringAnsi(LibRdKafka.name(rk));
logger(name, level, fac, buf);
};
LibRdKafka.conf_set_log_cb(config, LogDelegate);
StatsDelegate = (IntPtr rk, IntPtr json, UIntPtr json_len, IntPtr opaque) =>
{
OnStatistics?.Invoke(this, Marshal.PtrToStringAnsi(json));
return(0);
};
LibRdKafka.conf_set_stats_cb(config, StatsDelegate);
handle = SafeKafkaHandle.Create(type, config);
callbackCts = new CancellationTokenSource();
callbackTask = StartCallbackTask(callbackCts.Token);
}
internal static extern SafeTopicConfigHandle rd_kafka_default_topic_conf_dup(SafeKafkaHandle rk);
internal Producer(ProducerBuilder <TKey, TValue> builder)
{
var baseConfig = builder.ConstructBaseConfig(this);
// TODO: Make Tasks auto complete when EnableDeliveryReportsPropertyName is set to false.
// TODO: Hijack the "delivery.report.only.error" configuration parameter and add functionality to enforce that Tasks
// that never complete are never created when this is set to true.
this.statisticsHandler = baseConfig.statisticsHandler;
this.logHandler = baseConfig.logHandler;
this.errorHandler = baseConfig.errorHandler;
this.oAuthBearerTokenRefreshHandler = baseConfig.oAuthBearerTokenRefreshHandler;
this.partitioners = baseConfig.partitioners;
var config = Confluent.Kafka.Config.ExtractCancellationDelayMaxMs(baseConfig.config, out this.cancellationDelayMaxMs);
this.DeliveryReportCallback = DeliveryReportCallbackImpl;
Librdkafka.Initialize(null);
var modifiedConfig = Library.NameAndVersionConfig
.Concat(config
.Where(prop =>
prop.Key != ConfigPropertyNames.Producer.EnableBackgroundPoll &&
prop.Key != ConfigPropertyNames.Producer.EnableDeliveryReports &&
prop.Key != ConfigPropertyNames.Producer.DeliveryReportFields))
.ToList();
if (modifiedConfig.Where(obj => obj.Key == "delivery.report.only.error").Count() > 0)
{
// A managed object is kept alive over the duration of the produce request. If there is no
// delivery report generated, there will be a memory leak. We could possibly support this
// property by keeping track of delivery reports in managed code, but this seems like
// more trouble than it's worth.
throw new ArgumentException("The 'delivery.report.only.error' property is not supported by this client");
}
var enableBackgroundPollObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.EnableBackgroundPoll).Value;
if (enableBackgroundPollObj != null)
{
this.manualPoll = !bool.Parse(enableBackgroundPollObj);
}
var enableDeliveryReportsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.EnableDeliveryReports).Value;
if (enableDeliveryReportsObj != null)
{
this.enableDeliveryReports = bool.Parse(enableDeliveryReportsObj);
}
var deliveryReportEnabledFieldsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.DeliveryReportFields).Value;
if (deliveryReportEnabledFieldsObj != null)
{
var fields = deliveryReportEnabledFieldsObj.Replace(" ", "");
if (fields != "all")
{
this.enableDeliveryReportKey = false;
this.enableDeliveryReportValue = false;
this.enableDeliveryReportHeaders = false;
this.enableDeliveryReportTimestamp = false;
this.enableDeliveryReportPersistedStatus = false;
if (fields != "none")
{
var parts = fields.Split(',');
foreach (var part in parts)
{
switch (part)
{
case "key": this.enableDeliveryReportKey = true; break;
case "value": this.enableDeliveryReportValue = true; break;
case "timestamp": this.enableDeliveryReportTimestamp = true; break;
case "headers": this.enableDeliveryReportHeaders = true; break;
case "status": this.enableDeliveryReportPersistedStatus = true; break;
default: throw new ArgumentException(
$"Unknown delivery report field name '{part}' in config value '{ConfigPropertyNames.Producer.DeliveryReportFields}'.");
}
}
}
}
}
var configHandle = SafeConfigHandle.Create();
modifiedConfig.ForEach((kvp) =>
{
if (kvp.Value == null)
{
throw new ArgumentNullException($"'{kvp.Key}' configuration parameter must not be null.");
}
configHandle.Set(kvp.Key, kvp.Value);
});
IntPtr configPtr = configHandle.DangerousGetHandle();
if (enableDeliveryReports)
{
Librdkafka.conf_set_dr_msg_cb(configPtr, DeliveryReportCallback);
}
// Explicitly keep references to delegates so they are not reclaimed by the GC.
errorCallbackDelegate = ErrorCallback;
logCallbackDelegate = LogCallback;
statisticsCallbackDelegate = StatisticsCallback;
oAuthBearerTokenRefreshCallbackDelegate = OAuthBearerTokenRefreshCallback;
if (errorHandler != null)
{
Librdkafka.conf_set_error_cb(configPtr, errorCallbackDelegate);
}
if (logHandler != null)
{
Librdkafka.conf_set_log_cb(configPtr, logCallbackDelegate);
}
if (statisticsHandler != null)
{
Librdkafka.conf_set_stats_cb(configPtr, statisticsCallbackDelegate);
}
if (oAuthBearerTokenRefreshHandler != null)
{
Librdkafka.conf_set_oauthbearer_token_refresh_cb(configPtr, oAuthBearerTokenRefreshCallbackDelegate);
}
this.ownedKafkaHandle = SafeKafkaHandle.Create(RdKafkaType.Producer, configPtr, this);
if (this.partitioners?.Any() ?? false)
{
foreach (var partitioner in this.partitioners)
{
var topicConfigHandle = SafeTopicConfigHandle.Create();
IntPtr topicConfigPtr = topicConfigHandle.DangerousGetHandle();
Librdkafka.PartitionerDelegate partitionerDelegate =
(IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt,
IntPtr rkt_opaque, IntPtr msg_opaque) =>
{
if (this.ownedKafkaHandle.IsClosed)
{
return(Partition.Any);
}
var topic = partitioner.Key;
var providedPartitioner = partitioner.Value;
return(CallCustomPartitioner(topic, providedPartitioner, keydata, keylen, partition_cnt, rkt_opaque, msg_opaque));
};
this.partitionerCallbacks.Add(partitioner.Key, partitionerDelegate);
// Set partitioner on the topic_conf...
Librdkafka.topic_conf_set_partitioner_cb(topicConfigPtr, partitionerDelegate);
// Associate topic_conf with topic
// this also caches the topic handle (and topic_conf)
this.ownedKafkaHandle.getKafkaTopicHandle(partitioner.Key, topicConfigPtr);
// topic_conf ownership was transferred
topicConfigHandle.SetHandleAsInvalid();
}
}
configHandle.SetHandleAsInvalid(); // config object is no longer usable.
if (!manualPoll)
{
callbackCts = new CancellationTokenSource();
callbackTask = StartPollTask(callbackCts.Token);
}
InitializeSerializers(
builder.KeySerializer, builder.ValueSerializer,
builder.AsyncKeySerializer, builder.AsyncValueSerializer);
}
internal Producer(ProducerBuilder <TKey, TValue> builder)
{
var baseConfig = builder.ConstructBaseConfig(this);
// TODO: Make Tasks auto complete when EnableDeliveryReportsPropertyName is set to false.
// TODO: Hijack the "delivery.report.only.error" configuration parameter and add functionality to enforce that Tasks
// that never complete are never created when this is set to true.
this.statisticsHandler = baseConfig.statisticsHandler;
this.logHandler = baseConfig.logHandler;
this.errorHandler = baseConfig.errorHandler;
var config = Confluent.Kafka.Config.ExtractCancellationDelayMaxMs(baseConfig.config, out this.cancellationDelayMaxMs);
this.DeliveryReportCallback = DeliveryReportCallbackImpl;
Librdkafka.Initialize(null);
var modifiedConfig = config
.Where(prop =>
prop.Key != ConfigPropertyNames.Producer.EnableBackgroundPoll &&
prop.Key != ConfigPropertyNames.Producer.EnableDeliveryReports &&
prop.Key != ConfigPropertyNames.Producer.DeliveryReportFields);
if (modifiedConfig.Where(obj => obj.Key == "delivery.report.only.error").Count() > 0)
{
// A managed object is kept alive over the duration of the produce request. If there is no
// delivery report generated, there will be a memory leak. We could possibly support this
// property by keeping track of delivery reports in managed code, but this seems like
// more trouble than it's worth.
throw new ArgumentException("The 'delivery.report.only.error' property is not supported by this client");
}
var enableBackgroundPollObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.EnableBackgroundPoll).Value;
if (enableBackgroundPollObj != null)
{
this.manualPoll = !bool.Parse(enableBackgroundPollObj);
}
var enableDeliveryReportsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.EnableDeliveryReports).Value;
if (enableDeliveryReportsObj != null)
{
this.enableDeliveryReports = bool.Parse(enableDeliveryReportsObj);
}
var deliveryReportEnabledFieldsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.DeliveryReportFields).Value;
if (deliveryReportEnabledFieldsObj != null)
{
var fields = deliveryReportEnabledFieldsObj.Replace(" ", "");
if (fields != "all")
{
this.enableDeliveryReportKey = false;
this.enableDeliveryReportValue = false;
this.enableDeliveryReportHeaders = false;
this.enableDeliveryReportTimestamp = false;
this.enableDeliveryReportPersistedStatus = false;
if (fields != "none")
{
var parts = fields.Split(',');
foreach (var part in parts)
{
switch (part)
{
case "key": this.enableDeliveryReportKey = true; break;
case "value": this.enableDeliveryReportValue = true; break;
case "timestamp": this.enableDeliveryReportTimestamp = true; break;
case "headers": this.enableDeliveryReportHeaders = true; break;
case "status": this.enableDeliveryReportPersistedStatus = true; break;
default: throw new ArgumentException(
$"Unknown delivery report field name '{part}' in config value '{ConfigPropertyNames.Producer.DeliveryReportFields}'.");
}
}
}
}
}
var configHandle = SafeConfigHandle.Create();
modifiedConfig.ToList().ForEach((kvp) => {
if (kvp.Value == null)
{
throw new ArgumentNullException($"'{kvp.Key}' configuration parameter must not be null.");
}
configHandle.Set(kvp.Key, kvp.Value);
});
IntPtr configPtr = configHandle.DangerousGetHandle();
if (enableDeliveryReports)
{
Librdkafka.conf_set_dr_msg_cb(configPtr, DeliveryReportCallback);
}
// Explicitly keep references to delegates so they are not reclaimed by the GC.
errorCallbackDelegate = ErrorCallback;
logCallbackDelegate = LogCallback;
statisticsCallbackDelegate = StatisticsCallback;
Librdkafka.conf_set_error_cb(configPtr, errorCallbackDelegate);
Librdkafka.conf_set_log_cb(configPtr, logCallbackDelegate);
Librdkafka.conf_set_stats_cb(configPtr, statisticsCallbackDelegate);
this.ownedKafkaHandle = SafeKafkaHandle.Create(RdKafkaType.Producer, configPtr, this);
configHandle.SetHandleAsInvalid(); // config object is no longer useable.
if (!manualPoll)
{
callbackCts = new CancellationTokenSource();
callbackTask = StartPollTask(callbackCts.Token);
}
InitializeSerializers(
builder.KeySerializer, builder.ValueSerializer,
builder.AsyncKeySerializer, builder.AsyncValueSerializer);
}
/// <summary>
/// Creates a new <see cref="Confluent.Kafka.ConsumerBase" /> instance.
/// </summary>
/// <param name="config">
/// A collection of librdkafka configuration parameters
/// (refer to https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md)
/// and parameters specific to this client (refer to:
/// <see cref="Confluent.Kafka.ConfigPropertyNames" />).
/// At a minimum, 'bootstrap.servers' and 'group.id' must be
/// specified.
/// </param>
public ConsumerBase(IEnumerable <KeyValuePair <string, string> > config)
{
Librdkafka.Initialize(null);
config = Config.GetCancellationDelayMaxMs(config, out this.cancellationDelayMaxMs);
if (config.FirstOrDefault(prop => string.Equals(prop.Key, "group.id", StringComparison.Ordinal)).Value == null)
{
throw new ArgumentException("'group.id' configuration parameter is required and was not specified.");
}
var modifiedConfig = config
.Where(prop => prop.Key != ConfigPropertyNames.Consumer.ConsumeResultFields);
var enabledFieldsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Consumer.ConsumeResultFields).Value;
if (enabledFieldsObj != null)
{
var fields = enabledFieldsObj.ToString().Replace(" ", "");
if (fields != "all")
{
this.enableHeaderMarshaling = false;
this.enableTimestampMarshaling = false;
this.enableTopicNameMarshaling = false;
if (fields != "none")
{
var parts = fields.Split(',');
foreach (var part in parts)
{
switch (part)
{
case "headers": this.enableHeaderMarshaling = true; break;
case "timestamp": this.enableTimestampMarshaling = true; break;
case "topic": this.enableTopicNameMarshaling = true; break;
default: throw new ArgumentException(
$"Unexpected consume result field name '{part}' in config value '{ConfigPropertyNames.Consumer.ConsumeResultFields}'.");
}
}
}
}
}
var configHandle = SafeConfigHandle.Create();
modifiedConfig
.ToList()
.ForEach((kvp) => {
if (kvp.Value == null)
{
throw new ArgumentException($"'{kvp.Key}' configuration parameter must not be null.");
}
configHandle.Set(kvp.Key, kvp.Value.ToString());
});
// Explicitly keep references to delegates so they are not reclaimed by the GC.
rebalanceDelegate = RebalanceCallback;
commitDelegate = CommitCallback;
errorCallbackDelegate = ErrorCallback;
logCallbackDelegate = LogCallback;
statsCallbackDelegate = StatsCallback;
IntPtr configPtr = configHandle.DangerousGetHandle();
Librdkafka.conf_set_rebalance_cb(configPtr, rebalanceDelegate);
Librdkafka.conf_set_offset_commit_cb(configPtr, commitDelegate);
Librdkafka.conf_set_error_cb(configPtr, errorCallbackDelegate);
Librdkafka.conf_set_log_cb(configPtr, logCallbackDelegate);
Librdkafka.conf_set_stats_cb(configPtr, statsCallbackDelegate);
this.kafkaHandle = SafeKafkaHandle.Create(RdKafkaType.Consumer, configPtr, this);
configHandle.SetHandleAsInvalid(); // config object is no longer useable.
var pollSetConsumerError = kafkaHandle.PollSetConsumer();
if (pollSetConsumerError != ErrorCode.NoError)
{
throw new KafkaException(new Error(pollSetConsumerError,
$"Failed to redirect the poll queue to consumer_poll queue: {ErrorCodeExtensions.GetReason(pollSetConsumerError)}"));
}
}
internal Producer(ProducerBuilder <TKey, TValue> builder)
{
var baseConfig = builder.ConstructBaseConfig(this);
var partitioners = baseConfig.partitioners;
var defaultPartitioner = baseConfig.defaultPartitioner;
// TODO: Make Tasks auto complete when EnableDeliveryReportsPropertyName is set to false.
// TODO: Hijack the "delivery.report.only.error" configuration parameter and add functionality to enforce that Tasks
// that never complete are never created when this is set to true.
this.statisticsHandler = baseConfig.statisticsHandler;
this.logHandler = baseConfig.logHandler;
this.errorHandler = baseConfig.errorHandler;
this.oAuthBearerTokenRefreshHandler = baseConfig.oAuthBearerTokenRefreshHandler;
var config = Confluent.Kafka.Config.ExtractCancellationDelayMaxMs(baseConfig.config, out this.cancellationDelayMaxMs);
this.DeliveryReportCallback = DeliveryReportCallbackImpl;
Librdkafka.Initialize(null);
var modifiedConfig = Library.NameAndVersionConfig
.Concat(config
.Where(prop =>
prop.Key != ConfigPropertyNames.Producer.EnableBackgroundPoll &&
prop.Key != ConfigPropertyNames.Producer.EnableDeliveryReports &&
prop.Key != ConfigPropertyNames.Producer.DeliveryReportFields))
.ToList();
if (modifiedConfig.Where(obj => obj.Key == "delivery.report.only.error").Count() > 0)
{
// A managed object is kept alive over the duration of the produce request. If there is no
// delivery report generated, there will be a memory leak. We could possibly support this
// property by keeping track of delivery reports in managed code, but this seems like
// more trouble than it's worth.
throw new ArgumentException("The 'delivery.report.only.error' property is not supported by this client");
}
var enableBackgroundPollObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.EnableBackgroundPoll).Value;
if (enableBackgroundPollObj != null)
{
this.manualPoll = !bool.Parse(enableBackgroundPollObj);
}
var enableDeliveryReportsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.EnableDeliveryReports).Value;
if (enableDeliveryReportsObj != null)
{
this.enableDeliveryReports = bool.Parse(enableDeliveryReportsObj);
}
var deliveryReportEnabledFieldsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Producer.DeliveryReportFields).Value;
if (deliveryReportEnabledFieldsObj != null)
{
var fields = deliveryReportEnabledFieldsObj.Replace(" ", "");
if (fields != "all")
{
this.enableDeliveryReportKey = false;
this.enableDeliveryReportValue = false;
this.enableDeliveryReportHeaders = false;
this.enableDeliveryReportTimestamp = false;
this.enableDeliveryReportPersistedStatus = false;
if (fields != "none")
{
var parts = fields.Split(',');
foreach (var part in parts)
{
switch (part)
{
case "key": this.enableDeliveryReportKey = true; break;
case "value": this.enableDeliveryReportValue = true; break;
case "timestamp": this.enableDeliveryReportTimestamp = true; break;
case "headers": this.enableDeliveryReportHeaders = true; break;
case "status": this.enableDeliveryReportPersistedStatus = true; break;
default: throw new ArgumentException(
$"Unknown delivery report field name '{part}' in config value '{ConfigPropertyNames.Producer.DeliveryReportFields}'.");
}
}
}
}
}
var configHandle = SafeConfigHandle.Create();
IntPtr configPtr = configHandle.DangerousGetHandle();
modifiedConfig.ForEach((kvp) =>
{
if (kvp.Value == null)
{
throw new ArgumentNullException($"'{kvp.Key}' configuration parameter must not be null.");
}
configHandle.Set(kvp.Key, kvp.Value);
});
if (enableDeliveryReports)
{
Librdkafka.conf_set_dr_msg_cb(configPtr, DeliveryReportCallback);
}
// Explicitly keep references to delegates so they are not reclaimed by the GC.
errorCallbackDelegate = ErrorCallback;
logCallbackDelegate = LogCallback;
statisticsCallbackDelegate = StatisticsCallback;
oAuthBearerTokenRefreshCallbackDelegate = OAuthBearerTokenRefreshCallback;
if (errorHandler != null)
{
Librdkafka.conf_set_error_cb(configPtr, errorCallbackDelegate);
}
if (logHandler != null)
{
Librdkafka.conf_set_log_cb(configPtr, logCallbackDelegate);
}
if (statisticsHandler != null)
{
Librdkafka.conf_set_stats_cb(configPtr, statisticsCallbackDelegate);
}
if (oAuthBearerTokenRefreshHandler != null)
{
Librdkafka.conf_set_oauthbearer_token_refresh_cb(configPtr, oAuthBearerTokenRefreshCallbackDelegate);
}
Action <SafeTopicConfigHandle, PartitionerDelegate> addPartitionerToTopicConfig = (topicConfigHandle, partitioner) =>
{
Librdkafka.PartitionerDelegate librdkafkaPartitioner = (IntPtr rkt, IntPtr keydata, UIntPtr keylen, int partition_cnt, IntPtr rkt_opaque, IntPtr msg_opaque) =>
{
unsafe
{
var topicNamePtr = Librdkafka.topic_name(rkt);
var topic = Util.Marshal.PtrToStringUTF8(topicNamePtr);
var keyIsNull = keydata == IntPtr.Zero;
var keyBytes = keyIsNull
? ReadOnlySpan <byte> .Empty
: new ReadOnlySpan <byte>(keydata.ToPointer(), (int)keylen);
return(partitioner(topic, partition_cnt, keyBytes, keyIsNull));
}
};
this.partitionerHandles.Add(GCHandle.Alloc(librdkafkaPartitioner));
Librdkafka.topic_conf_set_partitioner_cb(topicConfigHandle.DangerousGetHandle(), librdkafkaPartitioner);
};
// Configure the default custom partitioner.
if (defaultPartitioner != null)
{
// The default topic config may have been modified by topic-level
// configuraton parameters passed down from the top level config.
// If that's the case, duplicate the default topic config to avoid
// colobbering any already configured values.
var defaultTopicConfigHandle = configHandle.GetDefaultTopicConfig();
SafeTopicConfigHandle topicConfigHandle =
defaultTopicConfigHandle.DangerousGetHandle() != IntPtr.Zero
? defaultTopicConfigHandle.Duplicate()
: SafeTopicConfigHandle.Create();
addPartitionerToTopicConfig(topicConfigHandle, defaultPartitioner);
Librdkafka.conf_set_default_topic_conf(configPtr, topicConfigHandle.DangerousGetHandle());
}
this.ownedKafkaHandle = SafeKafkaHandle.Create(RdKafkaType.Producer, configPtr, this);
configHandle.SetHandleAsInvalid(); // ownership was transferred.
// Per-topic partitioners.
foreach (var partitioner in partitioners)
{
var topicConfigHandle = this.ownedKafkaHandle.DuplicateDefaultTopicConfig();
addPartitionerToTopicConfig(topicConfigHandle, partitioner.Value);
this.ownedKafkaHandle.newTopic(partitioner.Key, topicConfigHandle.DangerousGetHandle());
}
if (!manualPoll)
{
callbackCts = new CancellationTokenSource();
callbackTask = StartPollTask(callbackCts.Token);
}
InitializeSerializers(
builder.KeySerializer, builder.ValueSerializer,
builder.AsyncKeySerializer, builder.AsyncValueSerializer);
}
internal Consumer(ConsumerBuilder <TKey, TValue> builder)
{
var baseConfig = builder.ConstructBaseConfig(this);
this.statisticsHandler = baseConfig.statisticsHandler;
this.logHandler = baseConfig.logHandler;
this.errorHandler = baseConfig.errorHandler;
this.partitionsAssignedHandler = baseConfig.partitionsAssignedHandler;
this.partitionsRevokedHandler = baseConfig.partitionsRevokedHandler;
this.partitionsLostHandler = baseConfig.partitionsLostHandler;
this.offsetsCommittedHandler = baseConfig.offsetsCommittedHandler;
this.oAuthBearerTokenRefreshHandler = baseConfig.oAuthBearerTokenRefreshHandler;
this.revokedOrLostHandlerIsFunc = baseConfig.revokedOrLostHandlerIsFunc;
Librdkafka.Initialize(null);
var config = Confluent.Kafka.Config.ExtractCancellationDelayMaxMs(baseConfig.config, out this.cancellationDelayMaxMs);
if (config.FirstOrDefault(prop => string.Equals(prop.Key, "group.id", StringComparison.Ordinal)).Value == null)
{
throw new ArgumentException("'group.id' configuration parameter is required and was not specified.");
}
var modifiedConfig = Library.NameAndVersionConfig
.Concat(config.Where(prop => prop.Key != ConfigPropertyNames.Consumer.ConsumeResultFields))
.ToList();
var enabledFieldsObj = config.FirstOrDefault(prop => prop.Key == ConfigPropertyNames.Consumer.ConsumeResultFields).Value;
if (enabledFieldsObj != null)
{
var fields = enabledFieldsObj.Replace(" ", "");
if (fields != "all")
{
this.enableHeaderMarshaling = false;
this.enableTimestampMarshaling = false;
this.enableTopicNameMarshaling = false;
if (fields != "none")
{
var parts = fields.Split(',');
foreach (var part in parts)
{
switch (part)
{
case "headers": this.enableHeaderMarshaling = true; break;
case "timestamp": this.enableTimestampMarshaling = true; break;
case "topic": this.enableTopicNameMarshaling = true; break;
default: throw new ArgumentException(
$"Unexpected consume result field name '{part}' in config value '{ConfigPropertyNames.Consumer.ConsumeResultFields}'.");
}
}
}
}
}
var configHandle = SafeConfigHandle.Create();
modifiedConfig.ForEach((kvp) =>
{
if (kvp.Value == null)
{
throw new ArgumentNullException($"'{kvp.Key}' configuration parameter must not be null.");
}
configHandle.Set(kvp.Key, kvp.Value);
});
// Explicitly keep references to delegates so they are not reclaimed by the GC.
rebalanceDelegate = RebalanceCallback;
commitDelegate = CommitCallback;
errorCallbackDelegate = ErrorCallback;
logCallbackDelegate = LogCallback;
statisticsCallbackDelegate = StatisticsCallback;
oAuthBearerTokenRefreshCallbackDelegate = OAuthBearerTokenRefreshCallback;
IntPtr configPtr = configHandle.DangerousGetHandle();
if (partitionsAssignedHandler != null || partitionsRevokedHandler != null || partitionsLostHandler != null)
{
Librdkafka.conf_set_rebalance_cb(configPtr, rebalanceDelegate);
}
if (offsetsCommittedHandler != null)
{
Librdkafka.conf_set_offset_commit_cb(configPtr, commitDelegate);
}
if (errorHandler != null)
{
Librdkafka.conf_set_error_cb(configPtr, errorCallbackDelegate);
}
if (logHandler != null)
{
Librdkafka.conf_set_log_cb(configPtr, logCallbackDelegate);
}
if (statisticsHandler != null)
{
Librdkafka.conf_set_stats_cb(configPtr, statisticsCallbackDelegate);
}
if (oAuthBearerTokenRefreshHandler != null)
{
Librdkafka.conf_set_oauthbearer_token_refresh_cb(configPtr, oAuthBearerTokenRefreshCallbackDelegate);
}
this.kafkaHandle = SafeKafkaHandle.Create(RdKafkaType.Consumer, configPtr, this);
configHandle.SetHandleAsInvalid(); // config object is no longer useable.
var pollSetConsumerError = kafkaHandle.PollSetConsumer();
if (pollSetConsumerError != ErrorCode.NoError)
{
throw new KafkaException(new Error(pollSetConsumerError,
$"Failed to redirect the poll queue to consumer_poll queue: {ErrorCodeExtensions.GetReason(pollSetConsumerError)}"));
}
// setup key deserializer.
if (builder.KeyDeserializer == null)
{
if (!defaultDeserializers.TryGetValue(typeof(TKey), out object deserializer))
{
throw new InvalidOperationException(
$"Key deserializer was not specified and there is no default deserializer defined for type {typeof(TKey).Name}.");
}
this.keyDeserializer = (IDeserializer <TKey>)deserializer;
}
else
{
this.keyDeserializer = builder.KeyDeserializer;
}
// setup value deserializer.
if (builder.ValueDeserializer == null)
{
if (!defaultDeserializers.TryGetValue(typeof(TValue), out object deserializer))
{
throw new InvalidOperationException(
$"Value deserializer was not specified and there is no default deserializer defined for type {typeof(TValue).Name}.");
}
this.valueDeserializer = (IDeserializer <TValue>)deserializer;
}
else
{
this.valueDeserializer = builder.ValueDeserializer;
}
}
private void RebalanceCallback(
IntPtr rk,
ErrorCode err,
IntPtr partitionsPtr,
IntPtr opaque)
{
try
{
// Ensure registered handlers are never called as a side-effect of Dispose/Finalize (prevents deadlocks in common scenarios).
if (kafkaHandle.IsClosed)
{
// The RebalanceCallback should never be invoked as a side effect of Dispose.
// If for some reason flow of execution gets here, something is badly wrong.
// (and we have a closed librdkafka handle that is expecting an assign call...)
throw new Exception("Unexpected rebalance callback on disposed kafkaHandle");
}
if (kafkaHandle.RebalanceProtocol == "COOPERATIVE" &&
this.revokedOrLostHandlerIsFunc)
{
throw new InvalidOperationException("Neither revoked nor lost partition handlers may return an updated assignment when a COOPERATIVE assignor is in use");
}
var partitions = SafeKafkaHandle.GetTopicPartitionOffsetErrorList(partitionsPtr).Select(p => p.TopicPartition).ToList();
if (err == ErrorCode.Local_AssignPartitions)
{
if (partitionsAssignedHandler == null)
{
if (kafkaHandle.RebalanceProtocol == "COOPERATIVE")
{
IncrementalAssign(partitions.Select(p => new TopicPartitionOffset(p, Offset.Unset)));
}
else
{
Assign(partitions.Select(p => new TopicPartitionOffset(p, Offset.Unset)));
}
return;
}
lock (assignCallCountLockObj) { assignCallCount = 0; }
var assignTo = partitionsAssignedHandler(partitions);
lock (assignCallCountLockObj)
{
if (assignCallCount > 0)
{
throw new InvalidOperationException("(Incremental)Assign/Unassign must not be called in the partitions assigned handler");
}
}
if (kafkaHandle.RebalanceProtocol == "COOPERATIVE")
{
if (assignTo.Count() != partitions.Count())
{
throw new InvalidOperationException("The partitions assigned handler must not return a different set of topic partitions than it was provided");
}
var sortedPartitions = partitions.OrderBy(p => p).ToList();
var sortedAssignTo = assignTo.OrderBy(p => p.TopicPartition);
var partitionsIter = sortedPartitions.GetEnumerator();
foreach (var p in sortedAssignTo)
{
partitionsIter.MoveNext();
if (p.TopicPartition != partitionsIter.Current)
{
throw new InvalidOperationException("The partitions assigned handler must not return a different set of topic partitions than it was provided");
}
}
IncrementalAssign(sortedAssignTo);
}
else
{
Assign(assignTo);
}
return;
}
if (err == ErrorCode.Local_RevokePartitions)
{
if (partitionsRevokedHandler == null &&
(!kafkaHandle.AssignmentLost || partitionsLostHandler == null))
{
if (kafkaHandle.RebalanceProtocol == "COOPERATIVE")
{
IncrementalUnassign(partitions);
}
else
{
Unassign();
}
return;
}
var assignmentWithPositions = new List <TopicPartitionOffset>();
foreach (var tp in partitions)
{
try
{
assignmentWithPositions.Add(new TopicPartitionOffset(tp, Position(tp)));
}
catch
{
assignmentWithPositions.Add(new TopicPartitionOffset(tp, Offset.Unset));
}
}
lock (assignCallCountLockObj) { assignCallCount = 0; }
var assignTo = kafkaHandle.AssignmentLost
? (partitionsLostHandler != null
? partitionsLostHandler(assignmentWithPositions)
: partitionsRevokedHandler(assignmentWithPositions))
: partitionsRevokedHandler(assignmentWithPositions);
lock (assignCallCountLockObj)
{
if (assignCallCount > 0)
{
throw new InvalidOperationException("Assign/Unassign must not be called in the partitions revoked handler");
}
}
if (kafkaHandle.RebalanceProtocol == "COOPERATIVE")
{
// assignTo is always empty, not used in the COOPERATIVE case.
IncrementalUnassign(partitions);
}
else
{
Unassign();
}
return;
}
throw new KafkaException(kafkaHandle.CreatePossiblyFatalError(err, null));
}
catch (Exception e)
{
Unassign();
handlerException = e;
}
}
private void RebalanceCallback(
IntPtr rk,
ErrorCode err,
IntPtr partitions,
IntPtr opaque)
{
try
{
var partitionAssignment = SafeKafkaHandle.GetTopicPartitionOffsetErrorList(partitions).Select(p => p.TopicPartition).ToList();
// Ensure registered handlers are never called as a side-effect of Dispose/Finalize (prevents deadlocks in common scenarios).
if (kafkaHandle.IsClosed)
{
// The RebalanceCallback should never be invoked as a side effect of Dispose.
// If for some reason flow of execution gets here, something is badly wrong.
// (and we have a closed librdkafka handle that is expecting an assign call...)
throw new Exception("Unexpected rebalance callback on disposed kafkaHandle");
}
if (err == ErrorCode.Local_AssignPartitions)
{
if (partitionsAssignedHandler == null)
{
Assign(partitionAssignment.Select(p => new TopicPartitionOffset(p, Offset.Unset)));
return;
}
lock (assignCallCountLockObj) { assignCallCount = 0; }
var assignTo = partitionsAssignedHandler(partitionAssignment);
lock (assignCallCountLockObj)
{
if (assignCallCount > 0)
{
throw new InvalidOperationException("Assign/Unassign must not be called in the partitions assigned handler.");
}
}
Assign(assignTo);
return;
}
if (err == ErrorCode.Local_RevokePartitions)
{
if (partitionsRevokedHandler == null)
{
Unassign();
return;
}
var assignmentWithPositions = new List <TopicPartitionOffset>();
foreach (var tp in partitionAssignment)
{
try
{
assignmentWithPositions.Add(new TopicPartitionOffset(tp, Position(tp)));
}
catch
{
assignmentWithPositions.Add(new TopicPartitionOffset(tp, Offset.Unset));
}
}
lock (assignCallCountLockObj) { assignCallCount = 0; }
var assignTo = partitionsRevokedHandler(assignmentWithPositions);
lock (assignCallCountLockObj)
{
if (assignCallCount > 0)
{
throw new InvalidOperationException("Assign/Unassign must not be called in the partitions revoked handler.");
}
}
// This distinction is important because calling Assign whilst the consumer is being
// closed (which will generally trigger this callback) is disallowed.
if (assignTo.Count() > 0)
{
Assign(assignTo);
}
else
{
Unassign();
}
return;
}
throw new KafkaException(kafkaHandle.CreatePossiblyFatalError(err, null));
}
catch (Exception e)
{
handlerException = e;
}
}
private void RebalanceCallback(
IntPtr rk,
ErrorCode err,
IntPtr partitions,
IntPtr opaque)
{
var partitionList = SafeKafkaHandle.GetTopicPartitionOffsetErrorList(partitions).Select(p => p.TopicPartition).ToList();
// Ensure registered handlers are never called as a side-effect of Dispose/Finalize (prevents deadlocks in common scenarios).
if (kafkaHandle.IsClosed)
{
// The RebalanceCallback should never be invoked as a side effect of Dispose.
// If for some reason flow of execution gets here, something is badly wrong.
// (and we have a closed librdkafka handle that is expecting an assign call...)
throw new Exception("unexpected rebalance callback on disposed kafkaHandle");
}
// Note: The contract with librdkafka requires the application to acknowledge rebalances by calling Assign.
// To make the API less error prone, this is done automatically by the C# binding if required - the number
// of times assign is called by user code is tracked, and if this is zero, then assign is called automatically.
if (err == ErrorCode.Local_AssignPartitions)
{
var handler = OnPartitionsAssigned;
if (handler != null && handler.GetInvocationList().Length > 0)
{
assignCallCount = 0;
handler(this, partitionList);
if (assignCallCount == 1)
{
return;
}
if (assignCallCount > 1)
{
throw new InvalidOperationException(
$"Assign/Unassign was called {assignCallCount} times after OnPartitionsAssigned was raised. It must be called at most once.");
}
}
Assign(partitionList.Select(p => new TopicPartitionOffset(p, Offset.Invalid)));
}
else if (err == ErrorCode.Local_RevokePartitions)
{
var handler = OnPartitionsRevoked;
if (handler != null && handler.GetInvocationList().Length > 0)
{
assignCallCount = 0;
handler(this, partitionList);
if (assignCallCount == 1)
{
return;
}
if (assignCallCount > 1)
{
throw new InvalidOperationException(
$"Assign/Unassign was called {assignCallCount} times after OnPartitionsAssigned was raised. It must be called at most once.");
}
}
Unassign();
}
else
{
throw new KafkaException(kafkaHandle.CreatePossiblyFatalError(err, null));
}
}
