예제 #1
0
        public KafkaAdapterFactory(
            string name,
            KafkaStreamOptions options,
            SimpleQueueCacheOptions cacheOptions,
            SerializationManager serializationManager,
            ILoggerFactory loggerFactory,
            IGrainFactory grainFactory
            )
        {
            _options = options ?? throw new ArgumentNullException(nameof(options));

            _name = name;
            _serializationManager = serializationManager;
            _loggerFactory        = loggerFactory;
            _grainFactory         = grainFactory;
            _logger = loggerFactory.CreateLogger <KafkaAdapterFactory>();

            if (options.Topics != null && options.Topics.Count == 0)
            {
                throw new ArgumentNullException(nameof(options.Topics));
            }

            _adapterCache = new SimpleQueueAdapterCache(
                cacheOptions,
                name,
                loggerFactory
                );

            _queueProperties   = GetQueuesProperties();
            _streamQueueMapper = new ExternalQueueMapper(_queueProperties.Values);
        }
예제 #2
0
        /// <summary>
        /// Add Kafka Topic Stream from configuration.
        /// </summary>
        /// <param name="options">The <see cref="KafkaStreamOptions" /> used to configure Kafka streams.</param>
        /// <param name="domain">The domain of the stream.</param>
        /// <param name="streamType">The stream type.</param>
        /// <param name="version">The version of the stream.</param>
        /// <param name="configuration">The <see cref="IConfiguration" /> which contains Kafka Topic configuration options.</param>
        /// <returns>The <see cref="KafkaStreamOptions"/>. </returns>
        public static KafkaStreamOptions AddTopicStream([NotNull] this KafkaStreamOptions options,
                                                        [NotNull] string domain, [NotNull] string streamType, uint version, [NotNull] IConfiguration configuration)
        {
            if (options == null)
            {
                throw new ArgumentNullException(nameof(options));
            }
            if (string.IsNullOrWhiteSpace(domain))
            {
                throw new ArgumentException($"'{nameof(domain)}' must not be null, empty or whitespace.",
                                            nameof(domain));
            }
            if (string.IsNullOrWhiteSpace(streamType))
            {
                throw new ArgumentException($"'{nameof(streamType)}' must not be null, empty or whitespace.",
                                            nameof(streamType));
            }
            if (configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            return(AddTopicStream(options, StreamHelper.GetNamespace(domain, streamType, version),
                                  configuration));
        }
예제 #3
0
        /// <summary>
        /// Add Kafka Topic Stream from configuration section.
        /// </summary>
        /// <param name="options">The <see cref="KafkaStreamOptions" /> used to configure Kafka streams.</param>
        /// <param name="name">The name of the topic.</param>
        /// <param name="configurationSection">The <see cref="IConfigurationSection" /> which contains Kafka Topic configuration options.</param>
        /// <returns>The <see cref="KafkaStreamOptions"/>. </returns>
        public static KafkaStreamOptions AddTopicStream([NotNull] this KafkaStreamOptions options,
                                                        [NotNull] string name, [NotNull] IConfigurationSection configurationSection)
        {
            if (options == null)
            {
                throw new ArgumentNullException(nameof(options));
            }
            if (string.IsNullOrWhiteSpace(name))
            {
                throw new ArgumentException($"'{nameof(name)}' must not be null, empty or whitespace.", nameof(name));
            }

            var kafkaTopicOptions = configurationSection?.Get <KafkaTopicOptions>() ?? new KafkaTopicOptions();

            if (kafkaTopicOptions.Partitions < 1)
            {
                throw new ConfigurationErrorsException("Number of topic partitions cannot be less than 1.");
            }
            if (kafkaTopicOptions.ReplicationFactor < 1)
            {
                throw new ConfigurationErrorsException("Number of topic replicas cannot be less than 1.");
            }

            options.AddTopic(name, new TopicCreationConfig
            {
                Partitions          = kafkaTopicOptions.Partitions,
                AutoCreate          = kafkaTopicOptions.IsTopicCreationEnabled,
                ReplicationFactor   = kafkaTopicOptions.ReplicationFactor,
                RetentionPeriodInMs = kafkaTopicOptions.RetentionPeriodInMs
            });

            return(options);
        }
        public void FromConfiguration_CorrectSetup_ReturnsKafkaStreamOptions()
        {
            var config = new ConfigurationBuilder()
                         .AddJsonFile("appsettings.json")
                         .Build();
            var kafkaStreamOptions = new KafkaStreamOptions().FromConfiguration(config);

            Assert.NotNull(kafkaStreamOptions);
        }
        public KafkaAdapterReceiver(
            QueueProperties queueProperties,
            KafkaStreamOptions options,
            SerializationManager serializationManager,
            ILoggerFactory loggerFactory
            )
        {
            _options = options ?? throw new ArgumentNullException(nameof(options));

            _queueProperties      = queueProperties;
            _serializationManager = serializationManager;
            _logger = loggerFactory.CreateLogger <KafkaAdapterReceiver>();
        }
예제 #6
0
        /// <summary>
        /// Use KafkaStreamOptions from configuration.
        /// </summary>
        /// <param name="options">The <see cref="KafkaStreamOptions" /> used to configure Kafka streams.</param>
        /// <param name="configuration">The <see cref="IConfiguration" /> which contains Kafka Streams provider's configuration options.</param>
        /// <returns>The <see cref="KafkaStreamOptions"/> </returns>
        public static KafkaStreamOptions FromConfiguration([NotNull] this KafkaStreamOptions options,
                                                           [NotNull] IConfiguration configuration)
        {
            if (options == null)
            {
                throw new ArgumentNullException(nameof(options));
            }
            if (configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            return(options.FromConfiguration(configuration.GetSection(KafkaOptions.DefaultSectionName)));
        }
        public void AddTopicStream_KafkaStreamsConfigurationSectionNotNull_ShouldSetValuesOnKafkaStreamOptions()
        {
            var config = new ConfigurationBuilder()
                         .AddJsonFile("appsettings.json")
                         .Build();

            var expectedConfig     = config.GetSection(KafkaTopicOptions.DefaultSectionName).Get <KafkaTopicOptions>();
            var kafkaStreamOptions = new KafkaStreamOptions().AddTopicStream("Test", config.GetSection(KafkaTopicOptions.DefaultSectionName));

            Assert.NotNull(kafkaStreamOptions);
            Assert.Equal("Test", kafkaStreamOptions.Topics[0].Name);
            Assert.Equal(expectedConfig.Partitions, kafkaStreamOptions.Topics[0].Partitions);
            Assert.Equal(expectedConfig.ReplicationFactor, kafkaStreamOptions.Topics[0].ReplicationFactor);
            Assert.Equal(expectedConfig.RetentionPeriodInMs, kafkaStreamOptions.Topics[0].RetentionPeriodInMs);
            Assert.Equal(expectedConfig.IsTopicCreationEnabled, kafkaStreamOptions.Topics[0].AutoCreate);
        }
 public KafkaAdapterFactory(
     string name,
     KafkaStreamOptions options,
     SimpleQueueCacheOptions cacheOptions,
     SerializationManager serializationManager,
     ILoggerFactory loggerFactory,
     IGrainFactory grainFactory
     ) : this(name, options, cacheOptions, serializationManager, loggerFactory, grainFactory, null)
 {
     if (options.Topics.Any(topic => topic.IsExternal))
     {
         throw new InvalidOperationException(
                   "Cannot have external topic with no 'IExternalDeserializer' defined. Use 'AddJson' or 'AddAvro'"
                   );
     }
 }
예제 #9
0
        /// <summary>
        /// Use KafkaStreamOptions from configuration section.
        /// </summary>
        /// <param name="options">The <see cref="KafkaStreamOptions" /> used to configure Kafka streams.</param>
        /// <param name="configurationSection">The <see cref="IConfigurationSection" /> which contains Kafka Stream options.</param>
        /// <returns>The <see cref="KafkaStreamOptions"/> </returns>
        public static KafkaStreamOptions FromConfiguration([NotNull] this KafkaStreamOptions options,
                                                           [NotNull] IConfigurationSection configurationSection)
        {
            if (options == null)
            {
                throw new ArgumentNullException(nameof(options));
            }
            if (configurationSection?.Exists() != true)
            {
                throw new ConfigurationErrorsException(
                          $"Configuration section '{configurationSection?.Path}' is incorrect.");
            }

            var kafkaOptions = configurationSection.Get <KafkaOptions>();

            if (kafkaOptions == null)
            {
                throw new ConfigurationErrorsException(
                          $"Configuration section '{configurationSection.Path}' is incorrect.");
            }

            if (kafkaOptions.Brokers != null)
            {
                options.BrokerList = kafkaOptions.Brokers.Split(';');
            }

            options.ConsumerGroupId = kafkaOptions.ConsumerGroupId;

            if (kafkaOptions.Security.IsEnabled)
            {
                if (string.IsNullOrWhiteSpace(kafkaOptions.Security.SaslUsername))
                {
                    throw new ConfigurationErrorsException($"Security is enabled but username is empty");
                }
                if (string.IsNullOrEmpty(kafkaOptions.Security.SaslPassword))
                {
                    throw new ConfigurationErrorsException($"Security is enabled but password is empty");
                }

                options.SecurityProtocol = kafkaOptions.Security.SecurityProtocol;
                options.SaslUserName     = kafkaOptions.Security.SaslUsername;
                options.SaslPassword     = kafkaOptions.Security.SaslPassword;
                options.SaslMechanism    = kafkaOptions.Security.SaslMechanism;
            }

            return(options);
        }
예제 #10
0
        /// <summary>
        /// Add Kafka Topic Stream from configuration.
        /// </summary>
        /// <param name="options">The <see cref="KafkaStreamOptions" /> used to configure Kafka streams.</param>
        /// <param name="name">The name of the topic.</param>
        /// <param name="configuration">The <see cref="IConfiguration" /> which contains Kafka Topic configuration options.</param>
        /// <returns>The <see cref="KafkaStreamOptions"/>. </returns>
        public static KafkaStreamOptions AddTopicStream([NotNull] this KafkaStreamOptions options,
                                                        [NotNull] string name, [NotNull] IConfiguration configuration)
        {
            if (options == null)
            {
                throw new ArgumentNullException(nameof(options));
            }
            if (string.IsNullOrWhiteSpace(name))
            {
                throw new ArgumentException($"'{nameof(name)}' must not be null, empty or whitespace.", nameof(name));
            }
            if (configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            return(options.AddTopicStream(name, configuration.GetSection(KafkaTopicOptions.DefaultSectionName)));
        }
        public void FromConfiguration_SaslEnabled_ShouldSetValuesOnKafkaStreamOptions()
        {
            //Arrange
            var config = new ConfigurationBuilder()
                         .AddJsonFile("appsettings.json")
                         .Build();

            var expectedConfig = config.GetSection(KafkaOptions.DefaultSectionName).Get <KafkaOptions>();

            //Act
            var kafkaStreamOptions = new KafkaStreamOptions().FromConfiguration(config);

            //Assert
            Assert.NotNull(kafkaStreamOptions);
            Assert.Equal(expectedConfig.Security.SaslUsername, kafkaStreamOptions.SaslUserName);
            Assert.Equal(expectedConfig.Security.SaslPassword, kafkaStreamOptions.SaslPassword);
            Assert.Equal(expectedConfig.Security.SaslMechanism, kafkaStreamOptions.SaslMechanism);
            Assert.Equal(expectedConfig.Security.SecurityProtocol, kafkaStreamOptions.SecurityProtocol);
        }
        public KafkaAdapterReceiver(
            string providerName,
            QueueProperties queueProperties,
            KafkaStreamOptions options,
            SerializationManager serializationManager,
            ILoggerFactory loggerFactory,
            IGrainFactory grainFactory,
            IExternalStreamDeserializer externalDeserializer
            )
        {
            _options = options ?? throw new ArgumentNullException(nameof(options));

            _providerName         = providerName;
            _queueProperties      = queueProperties;
            _serializationManager = serializationManager;
            _grainFactory         = grainFactory;
            _externalDeserializer = externalDeserializer;
            _logger = loggerFactory.CreateLogger <KafkaAdapterReceiver>();
        }
예제 #13
0
        public static KafkaBatchContainer ToBatchContainer(
            this ConsumeResult <byte[], byte[]> result,
            SerializationManager serializationManager,
            KafkaStreamOptions options,
            string streamNamespace
            )
        {
            var externalHeader = result.Headers.FirstOrDefault(header => header.Key == options.ExternalMessageIdentifier);
            var sequence       = new EventSequenceTokenV2(result.Offset.Value);

            if (externalHeader != null)
            {
                var isExternal = BitConverter.ToBoolean(externalHeader.GetValueBytes(), 0);
                if (isExternal)
                {
                    var key = Encoding.UTF8.GetString(result.Key);
                    return(new KafkaBatchContainer(
                               StreamProviderUtils.GenerateStreamGuid(key),
                               streamNamespace,
                               new List <object> {
                        Encoding.UTF8.GetString(result.Value)
                    },
                               null,
                               isExternalBatch: true,
                               sequence,
                               result.TopicPartitionOffset
                               ));
                }
            }

            var batchContainer = serializationManager.DeserializeFromByteArray <KafkaBatchContainer>(result.Value);

            if (batchContainer.SequenceToken == null)
            {
                batchContainer.SequenceToken = sequence;
            }

            batchContainer.TopicPartitionOffSet = result.TopicPartitionOffset;

            return(batchContainer);
        }
예제 #14
0
        public KafkaAdapter(
            string providerName,
            KafkaStreamOptions options,
            IDictionary <string, QueueProperties> queueProperties,
            SerializationManager serializationManager,
            ILoggerFactory loggerFactory
            )
        {
            _options              = options;
            _queueProperties      = queueProperties;
            _serializationManager = serializationManager;
            _loggerFactory        = loggerFactory;
            _logger = _loggerFactory.CreateLogger <KafkaAdapter>();

            Name = providerName;

            _producer = new Producer <byte[], KafkaBatchContainer>(
                options.ToProducerProperties(),
                valueSerializer: (topic, container) => container.ToByteArray(_serializationManager)
                );
        }
예제 #15
0
        public KafkaAdapter(
            string providerName,
            KafkaStreamOptions options,
            IDictionary <string, QueueProperties> queueProperties,
            SerializationManager serializationManager,
            ILoggerFactory loggerFactory,
            IGrainFactory grainFactory
            )
        {
            _options              = options;
            _queueProperties      = queueProperties;
            _serializationManager = serializationManager;
            _loggerFactory        = loggerFactory;
            _grainFactory         = grainFactory;
            _logger = _loggerFactory.CreateLogger <KafkaAdapter>();

            Name = providerName;

            _producer = new ProducerBuilder <byte[], KafkaBatchContainer>(options.ToProducerProperties())
                        .SetValueSerializer(new KafkaBatchContainerSerializer(serializationManager))
                        .Build();
        }
        public KafkaAdapterFactory(
            string name,
            KafkaStreamOptions options,
            SimpleQueueCacheOptions cacheOptions,
            SerializationManager serializationManager,
            ILoggerFactory loggerFactory,
            IGrainFactory grainFactory,
            IExternalStreamDeserializer externalDeserializer
            )
        {
            _options = options ?? throw new ArgumentNullException(nameof(options));

            _name = name;
            _serializationManager = serializationManager;
            _loggerFactory        = loggerFactory;
            _grainFactory         = grainFactory;
            _externalDeserializer = externalDeserializer;
            _logger      = loggerFactory.CreateLogger <KafkaAdapterFactory>();
            _adminConfig = new AdminClientBuilder(options.ToAdminProperties());

            if (options.Topics != null && options.Topics.Count == 0)
            {
                throw new ArgumentNullException(nameof(options.Topics));
            }

            _adapterCache = new SimpleQueueAdapterCache(
                cacheOptions,
                name,
                loggerFactory
                );

            _queueProperties   = GetQueuesProperties().ToDictionary(q => q.QueueName);
            _streamQueueMapper = new ExternalQueueMapper(_queueProperties.Values);

            _config = _options.ToAdminProperties();
        }
        public void AddTopicStream_KafkaStreamsConfigurationSectionIsNull_ShouldSetDefaultValuesOnKafkaStreamOptions()
        {
            var kafkaStreamOptions = new KafkaStreamOptions().AddTopicStream("Test", null);

            Assert.NotNull(kafkaStreamOptions);
        }