public ProducerConfig GetProducerConfig(KafkaProducerEntity entity)
{
if (!AzureFunctionsFileHelper.TryGetValidFilePath(entity.Attribute.SslCertificateLocation, out var resolvedSslCertificationLocation))
{
resolvedSslCertificationLocation = entity.Attribute.SslCertificateLocation;
}
if (!AzureFunctionsFileHelper.TryGetValidFilePath(entity.Attribute.SslCaLocation, out var resolvedSslCaLocation))
{
resolvedSslCaLocation = entity.Attribute.SslCaLocation;
}
if (!AzureFunctionsFileHelper.TryGetValidFilePath(entity.Attribute.SslKeyLocation, out var resolvedSslKeyLocation))
{
resolvedSslKeyLocation = entity.Attribute.SslKeyLocation;
}
var kafkaOptions = this.config.Get <KafkaOptions>();
var conf = new ProducerConfig()
{
BootstrapServers = this.config.ResolveSecureSetting(nameResolver, entity.Attribute.BrokerList),
BatchNumMessages = entity.Attribute.BatchSize,
EnableIdempotence = entity.Attribute.EnableIdempotence,
MessageSendMaxRetries = entity.Attribute.MaxRetries,
MessageTimeoutMs = entity.Attribute.MessageTimeoutMs,
RequestTimeoutMs = entity.Attribute.RequestTimeoutMs,
SaslPassword = this.config.ResolveSecureSetting(nameResolver, entity.Attribute.Password),
SaslUsername = this.config.ResolveSecureSetting(nameResolver, entity.Attribute.Username),
SslKeyLocation = resolvedSslKeyLocation,
SslKeyPassword = entity.Attribute.SslKeyPassword,
SslCertificateLocation = resolvedSslCertificationLocation,
SslCaLocation = resolvedSslCaLocation,
Debug = kafkaOptions?.LibkafkaDebug,
MetadataMaxAgeMs = kafkaOptions?.MetadataMaxAgeMs,
SocketKeepaliveEnable = kafkaOptions?.SocketKeepaliveEnable
};
if (entity.Attribute.AuthenticationMode != BrokerAuthenticationMode.NotSet)
{
conf.SaslMechanism = (SaslMechanism)entity.Attribute.AuthenticationMode;
}
if (entity.Attribute.Protocol != BrokerProtocol.NotSet)
{
conf.SecurityProtocol = (SecurityProtocol)entity.Attribute.Protocol;
}
return(conf);
}
public Task StartAsync(CancellationToken cancellationToken)
{
AzureFunctionsFileHelper.InitializeLibrdKafka(this.logger);
var builder = this.CreateConsumerBuilder(GetConsumerConfiguration());
builder.SetErrorHandler((_, e) =>
{
logger.LogError(e.Reason);
})
.SetPartitionsAssignedHandler((_, e) =>
{
logger.LogInformation($"Assigned partitions: [{string.Join(", ", e)}]");
})
.SetPartitionsRevokedHandler((_, e) =>
{
logger.LogInformation($"Revoked partitions: [{string.Join(", ", e)}]");
});
if (ValueDeserializer != null)
{
builder.SetValueDeserializer(ValueDeserializer);
}
this.consumer = builder.Build();
var commitStrategy = new AsyncCommitStrategy <TKey, TValue>(consumer, this.logger);
functionExecutor = singleDispatch ?
(FunctionExecutorBase <TKey, TValue>) new SingleItemFunctionExecutor <TKey, TValue>(executor, consumer, this.options.ExecutorChannelCapacity, this.options.ChannelFullRetryIntervalInMs, commitStrategy, logger) :
new MultipleItemFunctionExecutor <TKey, TValue>(executor, consumer, this.options.ExecutorChannelCapacity, this.options.ChannelFullRetryIntervalInMs, commitStrategy, logger);
consumer.Subscribe(this.listenerConfiguration.Topic);
// Using a thread as opposed to a task since this will be long running
var thread = new Thread(ProcessSubscription)
{
IsBackground = true,
};
thread.Start(cancellationTokenSource.Token);
return(Task.CompletedTask);
}
string EnsureValidEventHubsCertificateLocation(string userProvidedLocation)
{
const string defaultEventhubsCertificateFilePath = "./cacert.pem";
if (!string.IsNullOrWhiteSpace(userProvidedLocation))
{
if (!AzureFunctionsFileHelper.TryGetValidFilePath(userProvidedLocation, out var validatedUserProvidedLocation))
{
throw new InvalidOperationException($"Could not find user provided event hubs certificate file '{userProvidedLocation}");
}
return(validatedUserProvidedLocation);
}
if (!AzureFunctionsFileHelper.TryGetValidFilePath(defaultEventhubsCertificateFilePath, out var validatedCertificateFilePath))
{
throw new InvalidOperationException($"Could not find event hubs certificate file '{defaultEventhubsCertificateFilePath}'");
}
return(validatedCertificateFilePath);
}
private ConsumerConfig GetConsumerConfiguration()
{
ConsumerConfig conf = new ConsumerConfig()
{
// enable auto-commit
EnableAutoCommit = true,
// disable auto storing read offsets since we need to store them after calling the trigger function
EnableAutoOffsetStore = false,
// Interval in which commits stored in memory will be saved
AutoCommitIntervalMs = this.options.AutoCommitIntervalMs,
// start from earliest if no checkpoint has been committed
AutoOffsetReset = AutoOffsetReset.Earliest,
// Secure communication/authentication
SaslMechanism = this.listenerConfiguration.SaslMechanism,
SaslUsername = this.listenerConfiguration.SaslUsername,
SaslPassword = this.listenerConfiguration.SaslPassword,
SecurityProtocol = this.listenerConfiguration.SecurityProtocol,
SslCaLocation = this.listenerConfiguration.SslCaLocation,
SslCertificateLocation = this.listenerConfiguration.SslCertificateLocation,
SslKeyLocation = this.listenerConfiguration.SslKeyLocation,
SslKeyPassword = this.listenerConfiguration.SslKeyPassword,
// Values from host configuration
StatisticsIntervalMs = this.options.StatisticsIntervalMs,
ReconnectBackoffMs = this.options.ReconnectBackoffMs,
ReconnectBackoffMaxMs = this.options.ReconnectBackoffMaxMs,
SessionTimeoutMs = this.options.SessionTimeoutMs,
MaxPollIntervalMs = this.options.MaxPollIntervalMs,
QueuedMinMessages = this.options.QueuedMinMessages,
QueuedMaxMessagesKbytes = this.options.QueuedMaxMessagesKbytes,
MaxPartitionFetchBytes = this.options.MaxPartitionFetchBytes,
FetchMaxBytes = this.options.FetchMaxBytes,
};
if (string.IsNullOrEmpty(this.listenerConfiguration.EventHubConnectionString))
{
// Setup native kafka configuration
conf.BootstrapServers = this.listenerConfiguration.BrokerList;
conf.GroupId = this.listenerConfiguration.ConsumerGroup;
if (!string.IsNullOrWhiteSpace(conf.SslCaLocation))
{
if (AzureFunctionsFileHelper.TryGetValidFilePath(conf.SslCaLocation, out var resolvedSslCaLocation))
{
this.logger.LogDebug("Found SslCaLocation in {filePath}", resolvedSslCaLocation);
conf.SslCaLocation = resolvedSslCaLocation;
}
else
{
this.logger.LogWarning("Could not find valid file path for SslCaLocation {filePath}", conf.SslCaLocation);
}
}
}
else
{
// Setup eventhubs kafka head configuration
var ehBrokerList = this.listenerConfiguration.BrokerList;
if (!ehBrokerList.Contains(EventHubsBrokerListDns))
{
ehBrokerList = $"{this.listenerConfiguration.BrokerList}{EventHubsBrokerListDns}:{EventHubsBrokerListPort}";
}
var consumerGroupToUse = string.IsNullOrEmpty(this.listenerConfiguration.ConsumerGroup) ? "$Default" : this.listenerConfiguration.ConsumerGroup;
conf.BootstrapServers = ehBrokerList;
conf.SecurityProtocol = SecurityProtocol.SaslSsl;
conf.SaslMechanism = SaslMechanism.Plain;
conf.SaslUsername = EventHubsSaslUsername;
conf.SaslPassword = this.listenerConfiguration.EventHubConnectionString;
conf.SslCaLocation = this.EnsureValidEventHubsCertificateLocation(this.listenerConfiguration.SslCaLocation);
conf.GroupId = consumerGroupToUse;
conf.BrokerVersionFallback = EventHubsBrokerVersionFallback;
}
return(conf);
}