public ProducerConfig GetProducerConfig(KafkaProducerEntity entity) { if (!AzureFunctionsFileHelper.TryGetValidFilePath(entity.Attribute.SslCertificateLocation, out var resolvedSslCertificationLocation)) { resolvedSslCertificationLocation = entity.Attribute.SslCertificateLocation; } if (!AzureFunctionsFileHelper.TryGetValidFilePath(entity.Attribute.SslCaLocation, out var resolvedSslCaLocation)) { resolvedSslCaLocation = entity.Attribute.SslCaLocation; } if (!AzureFunctionsFileHelper.TryGetValidFilePath(entity.Attribute.SslKeyLocation, out var resolvedSslKeyLocation)) { resolvedSslKeyLocation = entity.Attribute.SslKeyLocation; } var kafkaOptions = this.config.Get <KafkaOptions>(); var conf = new ProducerConfig() { BootstrapServers = this.config.ResolveSecureSetting(nameResolver, entity.Attribute.BrokerList), BatchNumMessages = entity.Attribute.BatchSize, EnableIdempotence = entity.Attribute.EnableIdempotence, MessageSendMaxRetries = entity.Attribute.MaxRetries, MessageTimeoutMs = entity.Attribute.MessageTimeoutMs, RequestTimeoutMs = entity.Attribute.RequestTimeoutMs, SaslPassword = this.config.ResolveSecureSetting(nameResolver, entity.Attribute.Password), SaslUsername = this.config.ResolveSecureSetting(nameResolver, entity.Attribute.Username), SslKeyLocation = resolvedSslKeyLocation, SslKeyPassword = entity.Attribute.SslKeyPassword, SslCertificateLocation = resolvedSslCertificationLocation, SslCaLocation = resolvedSslCaLocation, Debug = kafkaOptions?.LibkafkaDebug, MetadataMaxAgeMs = kafkaOptions?.MetadataMaxAgeMs, SocketKeepaliveEnable = kafkaOptions?.SocketKeepaliveEnable }; if (entity.Attribute.AuthenticationMode != BrokerAuthenticationMode.NotSet) { conf.SaslMechanism = (SaslMechanism)entity.Attribute.AuthenticationMode; } if (entity.Attribute.Protocol != BrokerProtocol.NotSet) { conf.SecurityProtocol = (SecurityProtocol)entity.Attribute.Protocol; } return(conf); }
string EnsureValidEventHubsCertificateLocation(string userProvidedLocation) { const string defaultEventhubsCertificateFilePath = "./cacert.pem"; if (!string.IsNullOrWhiteSpace(userProvidedLocation)) { if (!AzureFunctionsFileHelper.TryGetValidFilePath(userProvidedLocation, out var validatedUserProvidedLocation)) { throw new InvalidOperationException($"Could not find user provided event hubs certificate file '{userProvidedLocation}"); } return(validatedUserProvidedLocation); } if (!AzureFunctionsFileHelper.TryGetValidFilePath(defaultEventhubsCertificateFilePath, out var validatedCertificateFilePath)) { throw new InvalidOperationException($"Could not find event hubs certificate file '{defaultEventhubsCertificateFilePath}'"); } return(validatedCertificateFilePath); }
private ConsumerConfig GetConsumerConfiguration() { ConsumerConfig conf = new ConsumerConfig() { // enable auto-commit EnableAutoCommit = true, // disable auto storing read offsets since we need to store them after calling the trigger function EnableAutoOffsetStore = false, // Interval in which commits stored in memory will be saved AutoCommitIntervalMs = this.options.AutoCommitIntervalMs, // start from earliest if no checkpoint has been committed AutoOffsetReset = AutoOffsetReset.Earliest, // Secure communication/authentication SaslMechanism = this.listenerConfiguration.SaslMechanism, SaslUsername = this.listenerConfiguration.SaslUsername, SaslPassword = this.listenerConfiguration.SaslPassword, SecurityProtocol = this.listenerConfiguration.SecurityProtocol, SslCaLocation = this.listenerConfiguration.SslCaLocation, SslCertificateLocation = this.listenerConfiguration.SslCertificateLocation, SslKeyLocation = this.listenerConfiguration.SslKeyLocation, SslKeyPassword = this.listenerConfiguration.SslKeyPassword, // Values from host configuration StatisticsIntervalMs = this.options.StatisticsIntervalMs, ReconnectBackoffMs = this.options.ReconnectBackoffMs, ReconnectBackoffMaxMs = this.options.ReconnectBackoffMaxMs, SessionTimeoutMs = this.options.SessionTimeoutMs, MaxPollIntervalMs = this.options.MaxPollIntervalMs, QueuedMinMessages = this.options.QueuedMinMessages, QueuedMaxMessagesKbytes = this.options.QueuedMaxMessagesKbytes, MaxPartitionFetchBytes = this.options.MaxPartitionFetchBytes, FetchMaxBytes = this.options.FetchMaxBytes, }; if (string.IsNullOrEmpty(this.listenerConfiguration.EventHubConnectionString)) { // Setup native kafka configuration conf.BootstrapServers = this.listenerConfiguration.BrokerList; conf.GroupId = this.listenerConfiguration.ConsumerGroup; if (!string.IsNullOrWhiteSpace(conf.SslCaLocation)) { if (AzureFunctionsFileHelper.TryGetValidFilePath(conf.SslCaLocation, out var resolvedSslCaLocation)) { this.logger.LogDebug("Found SslCaLocation in {filePath}", resolvedSslCaLocation); conf.SslCaLocation = resolvedSslCaLocation; } else { this.logger.LogWarning("Could not find valid file path for SslCaLocation {filePath}", conf.SslCaLocation); } } } else { // Setup eventhubs kafka head configuration var ehBrokerList = this.listenerConfiguration.BrokerList; if (!ehBrokerList.Contains(EventHubsBrokerListDns)) { ehBrokerList = $"{this.listenerConfiguration.BrokerList}{EventHubsBrokerListDns}:{EventHubsBrokerListPort}"; } var consumerGroupToUse = string.IsNullOrEmpty(this.listenerConfiguration.ConsumerGroup) ? "$Default" : this.listenerConfiguration.ConsumerGroup; conf.BootstrapServers = ehBrokerList; conf.SecurityProtocol = SecurityProtocol.SaslSsl; conf.SaslMechanism = SaslMechanism.Plain; conf.SaslUsername = EventHubsSaslUsername; conf.SaslPassword = this.listenerConfiguration.EventHubConnectionString; conf.SslCaLocation = this.EnsureValidEventHubsCertificateLocation(this.listenerConfiguration.SslCaLocation); conf.GroupId = consumerGroupToUse; conf.BrokerVersionFallback = EventHubsBrokerVersionFallback; } return(conf); }