public ProducerConfig GetProducerConfig(KafkaAttribute attribute) { var conf = new ProducerConfig() { BootstrapServers = this.config.ResolveSecureSetting(nameResolver, attribute.BrokerList), BatchNumMessages = attribute.BatchSize, EnableIdempotence = attribute.EnableIdempotence, MessageSendMaxRetries = attribute.MaxRetries, MessageTimeoutMs = attribute.MessageTimeoutMs, RequestTimeoutMs = attribute.RequestTimeoutMs, SaslPassword = this.config.ResolveSecureSetting(nameResolver, attribute.Password), SaslUsername = this.config.ResolveSecureSetting(nameResolver, attribute.Username), SslKeyLocation = attribute.SslKeyLocation, }; if (attribute.AuthenticationMode != BrokerAuthenticationMode.NotSet) { conf.SaslMechanism = (SaslMechanism)attribute.AuthenticationMode; } if (attribute.Protocol != BrokerProtocol.NotSet) { conf.SecurityProtocol = (SecurityProtocol)attribute.Protocol; } return(conf); }
private IKafkaProducer Create(KafkaAttribute attribute, string brokerList) { Type keyType = attribute.KeyType ?? typeof(Null); Type valueType = attribute.ValueType; string avroSchema = null; if (valueType == null) { if (!string.IsNullOrEmpty(attribute.AvroSchema)) { avroSchema = attribute.AvroSchema; valueType = typeof(GenericRecord); } else { valueType = typeof(string); } } else { if (typeof(ISpecificRecord).IsAssignableFrom(valueType)) { var specificRecord = (ISpecificRecord)Activator.CreateInstance(valueType); avroSchema = specificRecord.Schema.ToString(); } } return((IKafkaProducer)Activator.CreateInstance( typeof(KafkaProducer <,>).MakeGenericType(keyType, valueType), this.GetProducerConfig(brokerList), avroSchema, this.loggerProvider.CreateLogger(LogCategories.CreateTriggerCategory("Kafka")))); }
private IKafkaProducer Create(Handle producerBaseHandle, KafkaAttribute attribute) { var valueType = SerializationHelper.GetValueType(attribute.ValueType, attribute.AvroSchema, null, out var avroSchema); var keyType = attribute.KeyType ?? typeof(Null); var valueSerializer = SerializationHelper.ResolveValueSerializer(valueType, attribute.AvroSchema); return((IKafkaProducer)Activator.CreateInstance( typeof(KafkaProducer <,>).MakeGenericType(keyType, valueType), producerBaseHandle, valueSerializer, loggerProvider.CreateLogger(LogCategories.CreateTriggerCategory("Kafka")))); }
public IKafkaProducer Create(KafkaAttribute attribute) { // Goal is to create as less producers as possible // We can group producers based on following criterias // - Broker List // - Configuration var producerConfig = this.GetProducerConfig(attribute); var producerKey = CreateKeyForConfig(producerConfig); var baseProducer = baseProducers.GetOrAdd(producerKey, (k) => CreateBaseProducer(producerConfig)); return(Create(baseProducer.Handle, attribute)); }
public IKafkaProducer Get(KafkaAttribute attribute) { var resolvedBrokerList = this.nameResolver.ResolveWholeString(attribute.BrokerList); var brokerListFromConfig = this.config.GetConnectionStringOrSetting(resolvedBrokerList); if (!string.IsNullOrEmpty(brokerListFromConfig)) { resolvedBrokerList = brokerListFromConfig; } var keyTypeName = attribute.KeyType == null ? string.Empty : attribute.KeyType.AssemblyQualifiedName; var valueTypeName = attribute.ValueType == null ? string.Empty : attribute.ValueType.AssemblyQualifiedName; var producerKey = $"{resolvedBrokerList}:keyTypeName:valueTypeName"; return(producers.GetOrAdd(producerKey, (k) => this.Create(attribute, resolvedBrokerList))); }
public KafkaAttributeBinding( string parameterName, KafkaAttribute attribute, IKafkaProducerFactory kafkaProducerFactory, IArgumentBinding <KafkaProducerEntity> argumentBinding, Type keyType, Type valueType, string avroSchema) { this.parameterName = parameterName; this.attribute = attribute ?? throw new ArgumentNullException(nameof(attribute)); this.kafkaProducerFactory = kafkaProducerFactory ?? throw new ArgumentNullException(nameof(kafkaProducerFactory)); this.argumentBinding = argumentBinding ?? throw new ArgumentNullException(nameof(argumentBinding)); this.keyType = keyType; this.valueType = valueType ?? throw new ArgumentNullException(nameof(valueType)); this.avroSchema = avroSchema; }
private IAsyncCollector <KafkaEventData> BuildCollectorFromAttribute(KafkaAttribute attribute) { return(new KafkaAsyncCollector(attribute.Topic, kafkaProducerManager.Get(attribute))); }