public ProducerConfig GetProducerConfig(KafkaAttribute attribute)
{
var conf = new ProducerConfig()
{
BootstrapServers = this.config.ResolveSecureSetting(nameResolver, attribute.BrokerList),
BatchNumMessages = attribute.BatchSize,
EnableIdempotence = attribute.EnableIdempotence,
MessageSendMaxRetries = attribute.MaxRetries,
MessageTimeoutMs = attribute.MessageTimeoutMs,
RequestTimeoutMs = attribute.RequestTimeoutMs,
SaslPassword = this.config.ResolveSecureSetting(nameResolver, attribute.Password),
SaslUsername = this.config.ResolveSecureSetting(nameResolver, attribute.Username),
SslKeyLocation = attribute.SslKeyLocation,
};
if (attribute.AuthenticationMode != BrokerAuthenticationMode.NotSet)
{
conf.SaslMechanism = (SaslMechanism)attribute.AuthenticationMode;
}
if (attribute.Protocol != BrokerProtocol.NotSet)
{
conf.SecurityProtocol = (SecurityProtocol)attribute.Protocol;
}
return(conf);
}
private IKafkaProducer Create(KafkaAttribute attribute, string brokerList)
{
Type keyType = attribute.KeyType ?? typeof(Null);
Type valueType = attribute.ValueType;
string avroSchema = null;
if (valueType == null)
{
if (!string.IsNullOrEmpty(attribute.AvroSchema))
{
avroSchema = attribute.AvroSchema;
valueType = typeof(GenericRecord);
}
else
{
valueType = typeof(string);
}
}
else
{
if (typeof(ISpecificRecord).IsAssignableFrom(valueType))
{
var specificRecord = (ISpecificRecord)Activator.CreateInstance(valueType);
avroSchema = specificRecord.Schema.ToString();
}
}
return((IKafkaProducer)Activator.CreateInstance(
typeof(KafkaProducer <,>).MakeGenericType(keyType, valueType),
this.GetProducerConfig(brokerList),
avroSchema,
this.loggerProvider.CreateLogger(LogCategories.CreateTriggerCategory("Kafka"))));
}
private IKafkaProducer Create(Handle producerBaseHandle, KafkaAttribute attribute)
{
var valueType = SerializationHelper.GetValueType(attribute.ValueType, attribute.AvroSchema, null, out var avroSchema);
var keyType = attribute.KeyType ?? typeof(Null);
var valueSerializer = SerializationHelper.ResolveValueSerializer(valueType, attribute.AvroSchema);
return((IKafkaProducer)Activator.CreateInstance(
typeof(KafkaProducer <,>).MakeGenericType(keyType, valueType),
producerBaseHandle,
valueSerializer,
loggerProvider.CreateLogger(LogCategories.CreateTriggerCategory("Kafka"))));
}
public IKafkaProducer Create(KafkaAttribute attribute)
{
// Goal is to create as less producers as possible
// We can group producers based on following criterias
// - Broker List
// - Configuration
var producerConfig = this.GetProducerConfig(attribute);
var producerKey = CreateKeyForConfig(producerConfig);
var baseProducer = baseProducers.GetOrAdd(producerKey, (k) => CreateBaseProducer(producerConfig));
return(Create(baseProducer.Handle, attribute));
}
public IKafkaProducer Get(KafkaAttribute attribute)
{
var resolvedBrokerList = this.nameResolver.ResolveWholeString(attribute.BrokerList);
var brokerListFromConfig = this.config.GetConnectionStringOrSetting(resolvedBrokerList);
if (!string.IsNullOrEmpty(brokerListFromConfig))
{
resolvedBrokerList = brokerListFromConfig;
}
var keyTypeName = attribute.KeyType == null ? string.Empty : attribute.KeyType.AssemblyQualifiedName;
var valueTypeName = attribute.ValueType == null ? string.Empty : attribute.ValueType.AssemblyQualifiedName;
var producerKey = $"{resolvedBrokerList}:keyTypeName:valueTypeName";
return(producers.GetOrAdd(producerKey, (k) => this.Create(attribute, resolvedBrokerList)));
}
public KafkaAttributeBinding(
string parameterName,
KafkaAttribute attribute,
IKafkaProducerFactory kafkaProducerFactory,
IArgumentBinding <KafkaProducerEntity> argumentBinding,
Type keyType,
Type valueType,
string avroSchema)
{
this.parameterName = parameterName;
this.attribute = attribute ?? throw new ArgumentNullException(nameof(attribute));
this.kafkaProducerFactory = kafkaProducerFactory ?? throw new ArgumentNullException(nameof(kafkaProducerFactory));
this.argumentBinding = argumentBinding ?? throw new ArgumentNullException(nameof(argumentBinding));
this.keyType = keyType;
this.valueType = valueType ?? throw new ArgumentNullException(nameof(valueType));
this.avroSchema = avroSchema;
}
private IAsyncCollector <KafkaEventData> BuildCollectorFromAttribute(KafkaAttribute attribute)
{
return(new KafkaAsyncCollector(attribute.Topic, kafkaProducerManager.Get(attribute)));
}