public ProducerConfig GetProducerConfig(KafkaProducerEntity entity)
{
var conf = new ProducerConfig()
{
BootstrapServers = this.config.ResolveSecureSetting(nameResolver, entity.Attribute.BrokerList),
BatchNumMessages = entity.Attribute.BatchSize,
EnableIdempotence = entity.Attribute.EnableIdempotence,
MessageSendMaxRetries = entity.Attribute.MaxRetries,
MessageTimeoutMs = entity.Attribute.MessageTimeoutMs,
RequestTimeoutMs = entity.Attribute.RequestTimeoutMs,
SaslPassword = this.config.ResolveSecureSetting(nameResolver, entity.Attribute.Password),
SaslUsername = this.config.ResolveSecureSetting(nameResolver, entity.Attribute.Username),
SslKeyLocation = entity.Attribute.SslKeyLocation,
SslKeyPassword = entity.Attribute.SslKeyPassword,
SslCertificateLocation = entity.Attribute.SslCertificateLocation,
SslCaLocation = entity.Attribute.SslCaLocation
};
if (entity.Attribute.AuthenticationMode != BrokerAuthenticationMode.NotSet)
{
conf.SaslMechanism = (SaslMechanism)entity.Attribute.AuthenticationMode;
}
if (entity.Attribute.Protocol != BrokerProtocol.NotSet)
{
conf.SecurityProtocol = (SecurityProtocol)entity.Attribute.Protocol;
}
return(conf);
}
public KafkaProducerAsyncCollector(KafkaProducerEntity entity, Guid functionInstanceId)
{
if (entity == null)
{
throw new ArgumentNullException("entity");
}
this.entity = entity;
this.functionInstanceId = functionInstanceId;
}
public CollectorValueProvider(KafkaProducerEntity entity, object value, Type valueType)
{
if (value != null && !valueType.IsAssignableFrom(value.GetType()))
{
throw new InvalidOperationException("value is not of the correct type.");
}
this.entity = entity;
this.value = value;
this.valueType = valueType;
}
public Task <IValueProvider> BindAsync(KafkaProducerEntity value, ValueBindingContext context)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
IValueProvider provider = new NonNullArrayConverterValueBinder <TArray>(value,
new SerializableTypeToKafkaEventDataConverter <TArray, TItem>(), context.FunctionInstanceId);
return(Task.FromResult(provider));
}
public Task <IValueProvider> BindAsync(KafkaProducerEntity value, ValueBindingContext context)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
IAsyncCollector <TItem> collector = new KafkaProducerAsyncCollector <TItem>(value, context.FunctionInstanceId);
IValueProvider provider = new CollectorValueProvider(value, collector, typeof(IAsyncCollector <TItem>));
return(Task.FromResult(provider));
}
private IKafkaProducer Create(Handle producerBaseHandle, KafkaProducerEntity entity)
{
var valueType = entity.ValueType ?? typeof(byte[]);
var keyType = entity.KeyType ?? typeof(Null);
var valueSerializer = SerializationHelper.ResolveValueSerializer(valueType, entity.AvroSchema);
return((IKafkaProducer)Activator.CreateInstance(
typeof(KafkaProducer <,>).MakeGenericType(keyType, valueType),
producerBaseHandle,
valueSerializer,
loggerProvider.CreateLogger(LogCategories.CreateTriggerCategory("Kafka"))));
}
public IKafkaProducer Create(KafkaProducerEntity entity)
{
// Goal is to create as less producers as possible
// We can group producers based on following criterias
// - Broker List
// - Configuration
var producerConfig = this.GetProducerConfig(entity);
var producerKey = CreateKeyForConfig(producerConfig);
var baseProducer = baseProducers.GetOrAdd(producerKey, (k) => CreateBaseProducer(producerConfig));
return(Create(baseProducer.Handle, entity));
}
public IKafkaProducer Create(KafkaProducerEntity entity)
{
AzureFunctionsFileHelper.InitializeLibrdKafka(this.loggerProvider.CreateLogger(LogCategories.CreateTriggerCategory("Kafka")));
// Goal is to create as less producers as possible
// We can group producers based on following criterias
// - Broker List
// - Configuration
var producerConfig = this.GetProducerConfig(entity);
var producerKey = CreateKeyForConfig(producerConfig);
var baseProducer = baseProducers.GetOrAdd(producerKey, (k) => CreateBaseProducer(producerConfig));
return(Create(baseProducer.Handle, entity));
}
public ProducerConfig GetProducerConfig(KafkaProducerEntity entity)
{
if (!AzureFunctionsFileHelper.TryGetValidFilePath(entity.Attribute.SslCertificateLocation, out var resolvedSslCertificationLocation))
{
resolvedSslCertificationLocation = entity.Attribute.SslCertificateLocation;
}
if (!AzureFunctionsFileHelper.TryGetValidFilePath(entity.Attribute.SslCaLocation, out var resolvedSslCaLocation))
{
resolvedSslCaLocation = entity.Attribute.SslCaLocation;
}
if (!AzureFunctionsFileHelper.TryGetValidFilePath(entity.Attribute.SslKeyLocation, out var resolvedSslKeyLocation))
{
resolvedSslKeyLocation = entity.Attribute.SslKeyLocation;
}
var kafkaOptions = this.config.Get <KafkaOptions>();
var conf = new ProducerConfig()
{
BootstrapServers = this.config.ResolveSecureSetting(nameResolver, entity.Attribute.BrokerList),
BatchNumMessages = entity.Attribute.BatchSize,
EnableIdempotence = entity.Attribute.EnableIdempotence,
MessageSendMaxRetries = entity.Attribute.MaxRetries,
MessageTimeoutMs = entity.Attribute.MessageTimeoutMs,
RequestTimeoutMs = entity.Attribute.RequestTimeoutMs,
SaslPassword = this.config.ResolveSecureSetting(nameResolver, entity.Attribute.Password),
SaslUsername = this.config.ResolveSecureSetting(nameResolver, entity.Attribute.Username),
SslKeyLocation = resolvedSslKeyLocation,
SslKeyPassword = entity.Attribute.SslKeyPassword,
SslCertificateLocation = resolvedSslCertificationLocation,
SslCaLocation = resolvedSslCaLocation,
Debug = kafkaOptions?.LibkafkaDebug,
MetadataMaxAgeMs = kafkaOptions?.MetadataMaxAgeMs,
SocketKeepaliveEnable = kafkaOptions?.SocketKeepaliveEnable
};
if (entity.Attribute.AuthenticationMode != BrokerAuthenticationMode.NotSet)
{
conf.SaslMechanism = (SaslMechanism)entity.Attribute.AuthenticationMode;
}
if (entity.Attribute.Protocol != BrokerProtocol.NotSet)
{
conf.SecurityProtocol = (SecurityProtocol)entity.Attribute.Protocol;
}
return(conf);
}
public async Task <IValueProvider> BindAsync(BindingContext context)
{
context.CancellationToken.ThrowIfCancellationRequested();
var entity = new KafkaProducerEntity
{
KafkaProducerFactory = this.kafkaProducerFactory,
KeyType = this.keyType ?? typeof(Null),
ValueType = this.valueType,
Topic = this.attribute.Topic,
Attribute = this.attribute,
AvroSchema = this.avroSchema,
};
return(await BindAsync(entity, context.ValueContext));
}
public async Task <IValueProvider> BindAsync(object value, ValueBindingContext context)
{
context.CancellationToken.ThrowIfCancellationRequested();
var entity = new KafkaProducerEntity
{
KafkaProducerFactory = this.kafkaProducerFactory,
KeyType = this.keyType ?? typeof(Null),
ValueType = this.valueType,
Topic = this.config.ResolveSecureSetting(this.nameResolver, this.attribute.Topic),
Attribute = this.attribute,
AvroSchema = this.avroSchema,
};
return(await BindAsync(entity, context));
}
private Task <IValueProvider> BindAsync(KafkaProducerEntity value, ValueBindingContext context)
{
return(argumentBinding.BindAsync(value, context));
}
public NonNullArrayConverterValueBinder(KafkaProducerEntity entity, IConverter <T, IKafkaEventData[]> converter, Guid functionInstanceId)
{
this.entity = entity;
this.converter = converter;
this.functionInstanceId = functionInstanceId;
}
public KafkaEventDataValueBinder(KafkaProducerEntity entity, Guid functionInstanceId)
{
this.entity = entity;
this.functionInstanceId = functionInstanceId;
}
